diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..04c15a9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +site + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e46ace4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +# Start from existing PostreSQL 10 Debian image +# https://hub.docker.com/_/postgres/ +FROM postgres:10 + +MAINTAINER Mihai Criveti + +# Update and install PostGIS +RUN apt-get update +RUN apt-get install --no-install-recommends --yes postgresql-10-postgis-2.4 postgresql-10-postgis-2.4-scripts postgresql-contrib diff --git a/build-docker.sh b/build-docker.sh new file mode 100755 index 0000000..ccfda74 --- /dev/null +++ b/build-docker.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# Build and tag the image as cmihai/postgis +docker build --tag cmihai/postgis . diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..bf18147 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,27 @@ +version: '3.1' +services: + + postgis: + image: cmihai/postgis + container_name: postgis + ports: + - '5432:5432' + environment: + POSTGRES_PASSWORD: postgres + volumes: + - pgdata:/var/lib/postgresql/data + + pgadmin4: + image: dpage/pgadmin4 + container_name: pgadmin4 + ports: + - '5050:80' + environment: + PGADMIN_DEFAULT_EMAIL: admin + PGADMIN_DEFAULT_PASSWORD: admin + links: + - postgis + +volumes: + pgdata: + diff --git a/docs/about.md b/docs/about.md new file mode 100644 index 0000000..f1295c7 --- /dev/null +++ b/docs/about.md @@ -0,0 +1,6 @@ +# About Me + +## Mihai Criveti + +https://ie.linkedin.com/in/crivetimihai + diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..cf6030c --- /dev/null +++ b/docs/index.md @@ -0,0 +1,41 @@ +# Article Series: + +- Part 1: Setup PostGIS using Docker +- Part 2: Connecting QGIS to PostGIS and importing data +- Part 3: Querying Data +- Part 4: Advanced Analytics + + +### Terms and Tools: + + +| Term | Description | +|-----------------|--------------------------------------------------------------| +| GIS | Geographic Information System (manage, analyze and present geographic data) | +| Enterprise GIS | GIS system that spans the entire organization | +| Geospacial Database| Examples include PostgreSQL PostGIS, Oracle Spacial, SQLite's Spatialite or MSSQL Spacial | +| Desktop GIS | Examples include ESRI ArcGIS and QGIS | +| QGIS | Desktop GIS Software | +| PostgreSQL | Open Source RDBMS Server | +| PostGIS | Geospecial Extension to PostgreSQL | +| PgAdmin4 | PostgreSQL Web based SQL Editor and Administration Tool | +| Docker | We'll use the Docker container platform to run the software. | +| Docker Commpose | Compose multiple applications. | +| Docker Hub | Publish the image to docker hub. | + +### Links and Reference: + +- Github Repository with Dockerfile and docker-compose.yaml +- Docker Image: + + +### Reference and further documentation: + +- http://workshops.boundlessgeo.com/postgis-intro/ +- https://postgis.net/docs/manual-2.4/ +- https://boundlessgeo.com/gis-solutions/ + + +### Fancy math with Hilbert curves: + +- http://blog.christianperone.com/2015/08/googles-s2-geometry-on-the-sphere-cells-and-hilbert-curve/ diff --git a/docs/opendata/README.md b/docs/opendata/README.md new file mode 100644 index 0000000..50d0bc6 --- /dev/null +++ b/docs/opendata/README.md @@ -0,0 +1,40 @@ +# A look at Open Data + +## Common GIS data formats: + +QGIS can be used to import a variety of formats in PostGIS, including Shapefiles, KML and GeoJSON. + +- Shapefile: a popular geospatial vector data format for geographic information system (GIS) software. It is developed and regulated by Esri as a (mostly) open specification for data interoperability among Esri and other GIS software products. +- KML: Keyhole Markup Language is an XML notation for expressing geographic annotation and visualization within Internet-based, two-dimensional maps and three-dimensional Earth browsers. +- GeoJSON: a format for encoding a variety of geographic data structures. + + +## Open Data Sources +- Ireland's Open Data Portal: [https://data.gov.ie/](https://data.gov.ie/) +- Irish Spatial Data Exchange: [http://isde.ie](http://isde.ie) +- Census 2016 Open Data: [http://census2016.geohive.ie/](http://census2016.geohive.ie/) +- European Data Portal: [https://www.europeandataportal.eu](https://www.europeandataportal.eu) +- 2600+ Open Data sources: [https://www.opendatasoft.com/a-comprehensive-list-of-all-open-data-portals-around-the-world/](https://www.opendatasoft.com/a-comprehensive-list-of-all-open-data-portals-around-the-world/) + + +## Interesting datasets: + +1. [Landslide / Geohazards data, from GSI](https://www.gsi.ie/en-ie/data-and-maps/Pages/Geohazards.aspx) +2. [Railway stations](https://data.gov.ie/dataset/railway-stations-osi-national-250k-map-of-ireland) +3. [Railway lines](https://data.gov.ie/dataset/rail-network-osi-national-250k-map-of-ireland) +4. [Railway road intersections](https://data.gov.ie/dataset/road-rail-intersections-osi-national-250k-map-of-ireland) +5. [Built up areas](https://data.gov.ie/dataset/built-up-areas-osi-national-250k-map-of-ireland) +6. [Settlements](https://data.gov.ie/dataset/settlements-ungeneralised-osi-national-statistical-boundaries) +7. [Population](https://data.gov.ie/dataset/centres-of-population-osi-national-placenames-gazetteer) +8. [Groundwater vulnerability](https://data.gov.ie/dataset/gsi-groundwater-vulnerability) +9. [Dublin development plans](https://data.gov.ie/dataset/development-plans-dublin-city) + +## Downloading data: + +```bash +# Download the data: +wget http://spatial.dcenr.gov.ie/GSI_DOWNLOAD/Landslide_Susceptibility_Map_Ireland.zip + +# Unzip the data: +unzip Landslide_Susceptibility_Map_Ireland.zip +``` diff --git a/docs/psql/README.md b/docs/psql/README.md new file mode 100644 index 0000000..2fead75 --- /dev/null +++ b/docs/psql/README.md @@ -0,0 +1,38 @@ +# Using the `psql` commandline client and other tools + + +## List the networks. There should be a container running on the network `postgis_default` +``` +docker network ls +``` + +> 2e2afc387fbb postgis_default bridge local + + +## Connect to PostGIS using the `psql` client (from Docker): + +``` +docker run --net postgis_default -it --rm \ + --link postgis:postgis postgres psql -h postgis -U postgres -d gis +``` + +## Using a local client: +``` +psql -h localhost -U postgres -d gis +``` + +## List Tables: + +``` +\dt +``` + +## Run SQL Queries: + +``` +-- Events created by ABC, in descending order of creation date: +SELECT county, quaternary, slope_type, bedrock_ty, land_use_c, creationda, creator + FROM public."Landslide_Events" + WHERE creator = 'ABC' ORDER BY creationda DESC; +``` + diff --git a/docs/qgis/README.md b/docs/qgis/README.md new file mode 100644 index 0000000..8f1ea70 --- /dev/null +++ b/docs/qgis/README.md @@ -0,0 +1,58 @@ +# Loading data with QGIS + +In this article, I will show you how to: + +1. Install QGIS on Ubuntu Linux. +2. Connect QGIS to PostgreSQL/PostGIS. +3. Import data (shapefiles, GeoJSON) into the GIS database using DBManager. + + +## 1. Install QGIS: + +On Ubuntu Linux, you can use: + +```bash +sudo apt-get update +sudo apt-get install qgis +``` + +For other operating systems, follow the instructions listed at [https://qgis.org/en/site/](https://qgis.org/en/site/ +) + +### Connect to PostGIS +- Add PostGIS in QGis + +1. Under *Browser, Right click *PostGIS* > *New Connection* and select Name: postgis, Host: localhost, Port: 5432. +2. Save the connection details. + +![qgis-connect.png](qgis-connect.png) + + +## 2. Import Landslide shapefile data into QGIS, then Posgres + +### Import the data in QGIS + +> First, we import the sample data into QGIS: + +1. 'Layer > Data Source Manager > Home' and find the layers +2. Select the layers you wish to add and click the Add Selected Layers button. + +### Export the data to PostgreSQL / PostGIS +1. Click on *Database > DB Manager > DB Manager* +2. Select *PostGIS - yourdb > your schema > Table > Import Layer / File and name it (ex: Landslide_Events) +3. Repeat step 2 for every layer you wish to import +4. Close the DB Manager + +![db-manager.png](db-manager.png) + +> Note: The import activity can take a long time. You can monitor progress in the PGAdmin4 Dashboard, by looking at the `Tuples In: Inserts` graph: + +![load-performance.png](load-performance.png) + + +### Delete the layers - and load them from the DB + +1. In Layers, *right click each layer - Remove* +2. In *Browser > Postgis > postgis > public* - double click each layer (in the right order). + +![postgis-qgis.png](postgis-qgis.png) diff --git a/docs/qgis/db-manager.png b/docs/qgis/db-manager.png new file mode 100644 index 0000000..9cf13b1 Binary files /dev/null and b/docs/qgis/db-manager.png differ diff --git a/docs/qgis/load-performance.png b/docs/qgis/load-performance.png new file mode 100644 index 0000000..bd2176d Binary files /dev/null and b/docs/qgis/load-performance.png differ diff --git a/docs/qgis/postgis-qgis.png b/docs/qgis/postgis-qgis.png new file mode 100644 index 0000000..dae4e25 Binary files /dev/null and b/docs/qgis/postgis-qgis.png differ diff --git a/docs/qgis/qgis-connect.png b/docs/qgis/qgis-connect.png new file mode 100644 index 0000000..91b3a2c Binary files /dev/null and b/docs/qgis/qgis-connect.png differ diff --git a/docs/setup/README.md b/docs/setup/README.md new file mode 100644 index 0000000..d3cb69a --- /dev/null +++ b/docs/setup/README.md @@ -0,0 +1,155 @@ +# Docker PostGIS and PGAdmin + +In this article, I will show you how to: + +1. Create a Postgis docker image `FROM postgres` and publish it to [hub.docker.com](https://hub.docker.com/). +2. Create a Geospacial Database environment in Docker Compose with PostGIS and PGAdmin4. +3. Use PGAdmin4 to create a database an enable Geospacial extensions. + + +## 1. Build a PostGIS Docker Image + +- A `Dockerfile` contains instructions to build an image. +- Start by extending an existing PostgreSQL 10 Debian image: `FROM postgres:10`. +- Use `apt-get` to install required PostGIS extensions. +- Use `docker build` to create the image, then push it to docker hub. + +### Creating the `Dockerfile` + +```docker +# Extend exiting PostreSQL 10 Debian image: https://hub.docker.com/_/postgres/ +FROM postgres:10 + +MAINTAINER Mihai Criveti + +# Install PostGIS packages +RUN apt-get update +RUN apt-get install --no-install-recommends --yes \ + postgresql-10-postgis-2.4 postgresql-10-postgis-2.4-scripts postgresql-contrib +``` + +### Building the image + +- Turn the `Dockerfile` into a usable image using `docker build`. +- Tag the image with a namespace (the one used on Docker Hub): `cmihai` + +```bash +docker build --tag cmihai/postgis . +``` + +### Uploading the image to Docker Hub + +1. Push the Dockerfile, README and docker-compose.yaml examples to github +2. Test the image end to end +3. Push the image to docker hub + +```bash +export DOCKER_ID_USER="username" +docker login +docker push +docker tag cmihai/postgis $DOCKER_ID_USER/my_image +docker push $DOCKER_ID_USER/my_image +``` + + +## 2. Composing multiple images with docker compose + +- PGAdmin4 is a web based PostgreSQL Administration and SQL Development environment. +- Docker Compose can link an existing [dpage/pgadmin4](https://hub.docker.com/r/dpage/pgadmin4/) image from Docker Hub to `cmihai/postgis` +- Login to [http://localhost:5050](http://localhost:5050) `admin`:`admin` after running `docker-compose up` + + +### Create `docker-compose.yaml` + +```yaml +version: '3.1' +services: + + postgis: + image: cmihai/postgis + container_name: postgis + ports: + - '5432:5432' + environment: + POSTGRES_PASSWORD: postgres + volumes: + - pgdata:/var/lib/postgresql/data + + pgadmin4: + image: dpage/pgadmin4 + container_name: pgadmin4 + ports: + - '5050:80' + environment: + PGADMIN_DEFAULT_EMAIL: admin + PGADMIN_DEFAULT_PASSWORD: admin + links: + - postgis + +volumes: + pgdata: + +``` + + +### Starting the services + +Use `docker-compose up` to start the services: + +``` +docker-compose up +``` + +## 3. Create a database and enable PostGIS with PGAdmin4 + +1. Login to pgadmin4: [http://localhost:5050](http://localhost:5005) with admin:admin + +2. Add a connection to postgis with user/pass `postgres`:`postgres` ![create-server.png](create-server.png) + +3. Create a new database and call it `gis` ![create-database.png](create-database.png) + +4. Open the SQL Query Tool on the newly created `gis` database: +In the Browser window, select *Servers > postgis > Databases > gis*, the run *Tools > Query Tool* from the *Menu*. + +5. Run the following SQL code to enable postgis database extensions: + +```sql +-- Enable PostGIS (includes raster) +CREATE EXTENSION postgis; + +-- Enable Topology +CREATE EXTENSION postgis_topology; + +-- Enable PostGIS Advanced 3D and other geoprocessing algorithms +CREATE EXTENSION postgis_sfcgal; + +-- Fuzzy matching needed for Tiger +CREATE EXTENSION fuzzystrmatch; + +-- Rule based standardizer +CREATE EXTENSION address_standardizer; + +-- Example rule data set +CREATE EXTENSION address_standardizer_data_us; + +-- Enable US Tiger Geocoder +CREATE EXTENSION postgis_tiger_geocoder; +``` + +### Expected Outcome: `gis` database with geospacial extensions +- `Query returned successfully`: +![enable-postgis.png](enable-postgis.png) +- *gis > Extensions* now lists a number of GIS extensions: `postgis, postgis_sfgal, postgis_tiger_geocoder, postgis_topology, fuzzystrmatch, address_standardizer and address_standardizer_data_us`. +- *gis > Schema > public > Functions* has been populated with a high number (1000+) of GIS specific functions. +- A new table called `spacial_ref_sys` is now available under *gis > Schemas > public > Tables*. +- New schemas: `tiger`, `tiger_data` and `topology` have been created. + +### Next Steps: +- Load Geospacial data from shapefile, KML, GeoJSON, etc. +- Connect GIS Desktop clients such as QGIS. +- Connect to PostGIS using Python (ex: geopandas). +- Perform geospacial queries and analysis on the data. + +## Links and Reference: +- Github Repository with Dockerfile and docker-compose.yaml: [https://github.com/crivetimihai/geospacial-engineering](https://github.com/crivetimihai/geospacial-engineering) +- Docker Image: [https://hub.docker/com](https://hub.docker.com) diff --git a/docs/setup/create-database.png b/docs/setup/create-database.png new file mode 100644 index 0000000..2976963 Binary files /dev/null and b/docs/setup/create-database.png differ diff --git a/docs/setup/create-server.png b/docs/setup/create-server.png new file mode 100644 index 0000000..6738a57 Binary files /dev/null and b/docs/setup/create-server.png differ diff --git a/docs/setup/enable-postgis.png b/docs/setup/enable-postgis.png new file mode 100644 index 0000000..0a35eec Binary files /dev/null and b/docs/setup/enable-postgis.png differ diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..44dc1c3 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,13 @@ +site_name: GeoSpacial Engineering +site_author: Mihai Criveti +site_description: Geospacial Engineering with PostgreSQL, PostGIS, QGIS and Python Geopandas +repo_url: https://github.com/crivetimihai/geospacial-engineering + +theme: readthedocs + +pages: +- 'Geospacial Engineering with Docker, PostgreGIS and QGIS': 'index.md' +- 'Docker PostGIS and PGAdmin4': 'setup/README.md' +- 'Open Data Sources and Types': 'opendata/README.md' +- 'Load data with QGIS': 'qgis/README.md' +- 'Commandline and psql': 'psql/README.md' diff --git a/serve-site.sh b/serve-site.sh new file mode 100755 index 0000000..16e449a --- /dev/null +++ b/serve-site.sh @@ -0,0 +1 @@ +mkdocs serve --livereload -a 0.0.0.0:8000