From c370df4060c6901746c81d36df03fe4cd1e8fd86 Mon Sep 17 00:00:00 2001 From: dnomadb Date: Wed, 10 Jun 2020 00:00:06 -0700 Subject: [PATCH 1/2] demo for multipart --- mapbox_tilesets/scripts/cli.py | 60 ++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/mapbox_tilesets/scripts/cli.py b/mapbox_tilesets/scripts/cli.py index e568cc8..e6d4d75 100644 --- a/mapbox_tilesets/scripts/cli.py +++ b/mapbox_tilesets/scripts/cli.py @@ -6,6 +6,9 @@ import click import cligj +from contextlib import ExitStack +from functools import partial +from concurrent.futures import ThreadPoolExecutor from requests_toolbelt import MultipartEncoder import mapbox_tilesets @@ -445,6 +448,25 @@ def validate_source(features): click.echo("✔ valid") +def _upload_source(file, url=None): + """Upload an individual source part + """ + m = MultipartEncoder(fields={"file": ("file", file)}) + resp = requests.post( + url, + data=m, + headers={ + "Content-Disposition": "multipart/form-data", + "Content-type": m.content_type, + }, + ) + + if resp.status_code == 200: + return resp.json() + else: + raise errors.TilesetsError(resp.text) + + @cli.command("add-source") @click.argument("username", required=True, type=str) @click.argument("id", required=True, type=str) @@ -452,8 +474,13 @@ def validate_source(features): @click.option("--no-validation", is_flag=True, help="Bypass source file validation") @click.option("--token", "-t", required=False, type=str, help="Mapbox access token") @click.option("--indent", type=int, default=None, help="Indent for JSON output") +@click.option( + "--parts", default=1, type=click.IntRange(1, 10), help="Number of parts to use" +) @click.pass_context -def add_source(ctx, username, id, features, no_validation, token=None, indent=None): +def add_source( + ctx, username, id, features, no_validation, token=None, indent=None, parts=1 +): """Create/add a tileset source tilesets add-source @@ -464,27 +491,20 @@ def add_source(ctx, username, id, features, no_validation, token=None, indent=No f"{mapbox_api}/tilesets/v1/sources/{username}/{id}?access_token={mapbox_token}" ) - with tempfile.TemporaryFile() as file: - for feature in features: + with ExitStack() as task: + tmpfiles = [ + task.enter_context(tempfile.TemporaryFile()) for part in range(parts) + ] + for i, feature in enumerate(features): if not no_validation: utils.validate_geojson(feature) - file.write((json.dumps(feature) + "\n").encode("utf-8")) - - file.seek(0) - m = MultipartEncoder(fields={"file": ("file", file)}) - resp = requests.post( - url, - data=m, - headers={ - "Content-Disposition": "multipart/form-data", - "Content-type": m.content_type, - }, - ) - - if resp.status_code == 200: - click.echo(json.dumps(resp.json(), indent=indent)) - else: - raise errors.TilesetsError(resp.text) + tmpfiles[i % parts].write((json.dumps(feature) + "\n").encode("utf-8")) + for file in tmpfiles: + file.seek(0) + upload_source = partial(_upload_source, url=url) + with ThreadPoolExecutor(max_workers=4) as exec: + for resp in exec.map(upload_source, tmpfiles): + click.echo(json.dumps(resp, indent=indent)) @cli.command("view-source") From 9faf30689c458efd1856f240430ff638a974b226 Mon Sep 17 00:00:00 2001 From: dnomadb Date: Wed, 10 Jun 2020 16:55:38 -0700 Subject: [PATCH 2/2] delete before; delete after; delete often --- mapbox_tilesets/errors.py | 7 ++++++ mapbox_tilesets/scripts/cli.py | 41 +++++++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/mapbox_tilesets/errors.py b/mapbox_tilesets/errors.py index 34c4d87..83f94c4 100644 --- a/mapbox_tilesets/errors.py +++ b/mapbox_tilesets/errors.py @@ -25,3 +25,10 @@ class TilesetNameError(TilesetsError): def __init__(self, tileset_id): self.message = f"{tileset_id} is not a valid Tileset ID" + + +class SourceUploadFailed(TilesetsError): + """Source upload failed""" + + def __init__(self, message): + self.message = message diff --git a/mapbox_tilesets/scripts/cli.py b/mapbox_tilesets/scripts/cli.py index e6d4d75..095fe80 100644 --- a/mapbox_tilesets/scripts/cli.py +++ b/mapbox_tilesets/scripts/cli.py @@ -464,7 +464,7 @@ def _upload_source(file, url=None): if resp.status_code == 200: return resp.json() else: - raise errors.TilesetsError(resp.text) + raise errors.SourceUploadFailed(resp.text) @cli.command("add-source") @@ -503,8 +503,43 @@ def add_source( file.seek(0) upload_source = partial(_upload_source, url=url) with ThreadPoolExecutor(max_workers=4) as exec: - for resp in exec.map(upload_source, tmpfiles): - click.echo(json.dumps(resp, indent=indent)) + if parts > 1: + delete_url = "{0}/tilesets/v1/sources/{1}/{2}?access_token={3}".format( + mapbox_api, username, id, mapbox_token + ) + r = requests.delete(delete_url) + if r.status_code == 204: + pass + else: + raise errors.TilesetsError(r.text) + try: + uploaded = [i for i in exec.map(upload_source, tmpfiles)] + except errors.SourceUploadFailed as err: + if parts != 1: + url = "{0}/tilesets/v1/sources/{1}/{2}?access_token={3}".format( + mapbox_api, username, id, mapbox_token + ) + r = requests.delete(url) + if r.status_code == 204: + raise errors.TilesetsError( + "Multipart upload failed -- deleting extra parts" + ) + else: + raise errors.TilesetsError(r.text) + raise err + + if parts == 1: + click.echo(json.dumps(uploaded[0], indent=indent)) + else: + url = "{0}/tilesets/v1/sources/{1}/{2}?access_token={3}".format( + mapbox_api, username, id, mapbox_token + ) + + r = requests.get(url) + if r.status_code == 200: + click.echo(json.dumps(r.json(), indent=indent)) + else: + raise errors.TilesetsError(r.text) @cli.command("view-source")