From f217c564b7cbcf6333b4da4fbbf30d2e9dec5ba1 Mon Sep 17 00:00:00 2001 From: Fabien Del Olmo Date: Thu, 25 Feb 2021 09:06:19 +0100 Subject: [PATCH 1/3] mbtiler: use Wall mode and create index on tiles Wall mode speedup by about 10 the speed of writing in the database Create index on tiles for efficient access to this table --- rio_rgbify/mbtiler.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rio_rgbify/mbtiler.py b/rio_rgbify/mbtiler.py index f12a932..4367f18 100644 --- a/rio_rgbify/mbtiler.py +++ b/rio_rgbify/mbtiler.py @@ -317,6 +317,8 @@ def run(self, processes=4): # create a connection to the mbtiles file conn = sqlite3.connect(self.outpath) + # Wall mode : Speedup by 10 the speed of writing in the database + conn.execute('pragma journal_mode=wal') cur = conn.cursor() # create the tiles table @@ -325,6 +327,11 @@ def run(self, processes=4): "(zoom_level integer, tile_column integer, " "tile_row integer, tile_data blob);" ) + + # create index on tiles for efficient access to this table + cur.execute( + "CREATE UNIQUE INDEX IF NOT EXISTS tile_index on tiles (zoom_level, tile_column, tile_row);") + # create empty metadata cur.execute("CREATE TABLE metadata (name text, value text);") From e8432ae3afecdb8fe174371fcdcb5cf249cc3a3d Mon Sep 17 00:00:00 2001 From: Fabien Del Olmo Date: Thu, 25 Feb 2021 15:51:31 +0100 Subject: [PATCH 2/3] mbtiler : commit AFTER all insert --- rio_rgbify/mbtiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rio_rgbify/mbtiler.py b/rio_rgbify/mbtiler.py index 4367f18..4b32280 100644 --- a/rio_rgbify/mbtiler.py +++ b/rio_rgbify/mbtiler.py @@ -387,7 +387,7 @@ def run(self, processes=4): (z, x, tiley, buffer(contents)), ) - conn.commit() + conn.commit() conn.close() From 6b51e3668af7baec7284b8e96405ed0b6fc1afc7 Mon Sep 17 00:00:00 2001 From: Fabien Del Olmo Date: Thu, 25 Feb 2021 19:09:15 +0100 Subject: [PATCH 3/3] commit data every 1000 tiles (about 150 Mo) Otherwise, the file .mbtiles-wal becomes huge (same size as the final file). The result is the need to have twice the size of the final Mbtiles on the hard drive --- rio_rgbify/mbtiler.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rio_rgbify/mbtiler.py b/rio_rgbify/mbtiler.py index 4b32280..0bc0d64 100644 --- a/rio_rgbify/mbtiler.py +++ b/rio_rgbify/mbtiler.py @@ -373,6 +373,7 @@ def run(self, processes=4): constrained_bbox = list(mercantile.bounds(self.bounding_tile)) tiles = _make_tiles(constrained_bbox, "epsg:4326", self.min_z, self.max_z) + tilesCount = 0 for tile, contents in self.pool.imap_unordered(self.run_function, tiles): x, y, z = tile @@ -386,6 +387,11 @@ def run(self, processes=4): "VALUES (?, ?, ?, ?);", (z, x, tiley, buffer(contents)), ) + tilesCount = tilesCount + 1 + # commit data every 1000 tiles (about 150 Mo) + # Otherwise, the file .mbtiles-wal becomes huge (same size as the final file). The result is the need to have twice the size of the final Mbtiles on the hard drive + if (tilesCount % 1000 == 0): + conn.commit() conn.commit()