diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index 4f1224a..1da7fea 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -88,25 +88,25 @@ Changing the metadata is easy >>> rmapy.renew_token() True >>> collection = rmapy.get_meta_items() - >>> doc = [ d for d in collection if d.VissibleName == 'ModernC'][0] + >>> doc = [ d for d in collection if d.visibleName == 'ModernC'][0] >>> doc >>> doc.to_dict() - {'ID': 'a969fcd6-64b0-4f71-b1ce-d9533ec4a2a3', 'Version': 1, 'Message': '', 'Succes': True, 'BlobURLGet': '', 'BlobURLGetExpires': '0001-01-01T00:00:00Z', 'BlobURLPut': '', 'BlobURLPutExpires': '', 'ModifiedClient': '2019-09-18T20:12:07.206206Z', 'Type': 'DocumentType', 'VissibleName': 'ModernC', 'CurrentPage': 0, 'Bookmarked': False, 'Parent': ''} - >>> doc.VissibleName = "Modern C: The book of wisdom" + {'ID': 'a969fcd6-64b0-4f71-b1ce-d9533ec4a2a3', 'Version': 1, 'Message': '', 'Succes': True, 'BlobURLGet': '', 'BlobURLGetExpires': '0001-01-01T00:00:00Z', 'BlobURLPut': '', 'BlobURLPutExpires': '', 'ModifiedClient': '2019-09-18T20:12:07.206206Z', 'Type': 'DocumentType', 'visibleName': 'ModernC', 'CurrentPage': 0, 'Bookmarked': False, 'Parent': ''} + >>> doc.visibleName = "Modern C: The book of wisdom" >>> # push the changes back to the Remarkable Cloud ... rmapy.update_metadata(doc) True >>> collection = rmapy.get_meta_items() - >>> doc = [ d for d in docs if d.VissibleName == 'ModernC'][0] + >>> doc = [ d for d in docs if d.visibleName == 'ModernC'][0] Traceback (most recent call last): File "", line 1, in IndexError: list index out of range - >>> doc = [ d for d in docs if d.VissibleName == 'Modern C: The book of wisdom'][0] + >>> doc = [ d for d in docs if d.visibleName == 'Modern C: The book of wisdom'][0] >>> doc >>> doc.to_dict() - {'ID': 'a969fcd6-64b0-4f71-b1ce-d9533ec4a2a3', 'Version': 1, 'Message': '', 'Succes': True, 'BlobURLGet': '', 'BlobURLGetExpires': '0001-01-01T00:00:00Z', 'BlobURLPut': '', 'BlobURLPutExpires': '', 'ModifiedClient': '2019-09-18T20:12:07.206206Z', 'Type': 'DocumentType', 'VissibleName': 'Modern C: The book of wisdom', 'CurrentPage': 0, 'Bookmarked': False, 'Parent': ''} + {'ID': 'a969fcd6-64b0-4f71-b1ce-d9533ec4a2a3', 'Version': 1, 'Message': '', 'Succes': True, 'BlobURLGet': '', 'BlobURLGetExpires': '0001-01-01T00:00:00Z', 'BlobURLPut': '', 'BlobURLPutExpires': '', 'ModifiedClient': '2019-09-18T20:12:07.206206Z', 'Type': 'DocumentType', 'visibleName': 'Modern C: The book of wisdom', 'CurrentPage': 0, 'Bookmarked': False, 'Parent': ''} CollectionType @@ -145,9 +145,9 @@ Working with folders is easy! >>> rmapy.create_folder(new_folder) True >>> # verify - ... [ f for f in rmapy.get_meta_items() if f.VissibleName == "New Folder" ] + ... [ f for f in rmapy.get_meta_items() if f.visibleName == "New Folder" ] [] - >>> [ f for f in rmapy.get_meta_items() if f.VissibleName == "New Folder" ][0].ID == new_folder.ID + >>> [ f for f in rmapy.get_meta_items() if f.visibleName == "New Folder" ][0].ID == new_folder.ID True >>> # Move a document in a folder ... doc = rmapy.get_doc("a969fcd6-64b0-4f71-b1ce-d9533ec4a2a3") @@ -204,7 +204,7 @@ the remarkable file format: >>> rawDocument = ZipDocument(doc="/home/svancampenhout/27-11-2019.pdf") >>> rawDocument - >>> rawDocument.metadata["VissibleName"] + >>> rawDocument.metadata["visibleName"] '27-11-2019' Now we can upload this to a specific folder: @@ -213,7 +213,7 @@ Now we can upload this to a specific folder: :linenos: - >>> books = [ i for i in rm.get_meta_items() if i.VissibleName == "Boeken" ][0] + >>> books = [ i for i in rm.get_meta_items() if i.visibleName == "Boeken" ][0] >>> rm.upload(rawDocument, books) True @@ -222,6 +222,6 @@ And verify its existance: .. code-block:: python :linenos: - >>> [ i.VissibleName for i in collection.children(books) if i.Type == "DocumentType" ] + >>> [ i.visibleName for i in collection.children(books) if i.type == "DocumentType" ] ['Origin - Dan Brown', 'Flatland', 'Game Of Thrones', '27-11-2019'] diff --git a/rmapy/api.py b/rmapy/api.py index 56228e4..9788674 100644 --- a/rmapy/api.py +++ b/rmapy/api.py @@ -1,28 +1,81 @@ -import requests -from logging import getLogger from datetime import datetime +from logging import getLogger from typing import Union, Optional from uuid import uuid4 +from pathlib import Path + +import requests + from .collections import Collection -from .config import load, dump -from .document import Document, ZipDocument, from_request_stream -from .folder import Folder -from .exceptions import ( - AuthError, - DocumentNotFound, - ApiError, - UnsupportedTypeError,) +from .config import load, dump, config_cloudstatus_path from .const import (RFC3339Nano, USER_AGENT, BASE_URL, DEVICE_TOKEN_URL, USER_TOKEN_URL, - DEVICE,) + SERVICE_MGR_URL, + DEVICE, ) +from .document import Document, ZipDocument, from_request_stream +from .exceptions import ( + AuthError, + DocumentNotFound, + ApiError, + UnsupportedTypeError, ) +from .folder import Folder log = getLogger("rmapy") DocumentOrFolder = Union[Document, Folder] +class CloudStatus(object): + """Representation of the status report of the Remarkable Cloud + + """ + + def __init__(self, raw_cloudstatus: str) -> None: + self._raw_cloudstatus: str = "" + self._old_cloudstatus: str = "" + self.ID_dict: dict = {} + self.ID_dict_old: dict = {} + self.update(raw_cloudstatus) + + @staticmethod + def load() -> str: + cloudstatus = "\n\n" + if Path.exists(config_cloudstatus_path): + with open(config_cloudstatus_path, "r") as f: + cloudstatus = f.read() + + return cloudstatus + + def dump(self) -> None: + with open(config_cloudstatus_path, 'w') as f: + f.write(self._raw_cloudstatus) + + @staticmethod + def get_item_dict(cloudstatus: str) -> dict: + item_dict: dict = {} + for item in cloudstatus.split("\n")[1:-1]: + item_list = item.split(":") + cloud_id = item_list[0] + file_id = item_list[2] + item_dict[file_id] = cloud_id + + return item_dict + + def update(self, raw_cloudstatus: str) -> None: + self._raw_cloudstatus = raw_cloudstatus + self._old_cloudstatus = self.load() + self.dump() + + self.ID_dict: dict = self.get_item_dict(self._raw_cloudstatus) + self.ID_dict_old: dict = self.get_item_dict(self._old_cloudstatus) + + def get_new_and_updated(self, downloaded: list) -> dict: + return {key: val for key, val in self.ID_dict.items() if + key not in self.ID_dict_old or self.ID_dict_old[key] != val or key not in downloaded} + + class Client(object): """API Client for Remarkable Cloud @@ -35,6 +88,8 @@ class Client(object): "usertoken": "" } + base_url = None + def __init__(self): config = load() if "devicetoken" in config: @@ -42,6 +97,17 @@ def __init__(self): if "usertoken" in config: self.token_set["usertoken"] = config["usertoken"] + def get_base_url(self) -> str: + """Query the service url to get the latest document storage url""" + if self.base_url: + return self.base_url + + response = self.request("GET", SERVICE_MGR_URL + "/service/json/1/blob-storage?environment=production&apiVer=1") + + if response.ok: + self.base_url = response.json()["Host"] + return self.base_url + def request(self, method: str, path: str, data=None, body=None, headers=None, @@ -66,12 +132,6 @@ def request(self, method: str, path: str, if headers is None: headers = {} - if not path.startswith("http"): - if not path.startswith('/'): - path = '/' + path - url = f"{BASE_URL}{path}" - else: - url = path _headers = { "user-agent": USER_AGENT, @@ -82,8 +142,8 @@ def request(self, method: str, path: str, _headers["Authorization"] = f"Bearer {token}" for k in headers.keys(): _headers[k] = headers[k] - log.debug(url, _headers) - r = requests.request(method, url, + log.debug(path, _headers) + r = requests.request(method, path, json=body, data=data, headers=_headers, @@ -141,8 +201,8 @@ def renew_token(self): raise AuthError("Please register a device first") token = self.token_set["devicetoken"] response = self.request("POST", USER_TOKEN_URL, None, headers={ - "Authorization": f"Bearer {token}" - }) + "Authorization": f"Bearer {token}" + }) if response.ok: self.token_set["usertoken"] = response.text dump(self.token_set) @@ -163,6 +223,18 @@ def is_auth(self) -> bool: else: return False + def get_url_response(self, relative_path): + root_url_response = self.request("POST", self.base_url + "/api/v1/signed-urls/downloads", data=None, + body={"http_method": "GET", "relative_path": relative_path}) + root_url = root_url_response.json()["url"] + return self.request("GET", root_url) + + def get_root_dir(self): + """Request the root directory""" + self.get_base_url() + root = self.get_url_response("root") + return self.get_url_response(root.text).text + def get_meta_items(self) -> Collection: """Returns a new collection from meta items. @@ -173,14 +245,29 @@ def get_meta_items(self) -> Collection: Collection: a collection of Documents & Folders from the Remarkable Cloud """ - - response = self.request("GET", "/document-storage/json/2/docs") - collection = Collection() - log.debug(response.text) - for item in response.json(): - collection.add(item) - - return collection + cloudstatus = CloudStatus(self.get_root_dir()) + self._collection = Collection() + self._collection.load() + residuals = cloudstatus.get_new_and_updated([item["ID"] for item in self._collection.to_list()]) + + for meta_id, cloud_id in residuals.items(): + r = self.get_url_response(cloud_id) + for sub_item in r.iter_lines(): + sub_item_decoded = sub_item.decode("utf-8") + if ".metadata" in sub_item_decoded: + meta_cloud_id = sub_item_decoded.split(":")[0] + break + + meta_response = self.get_url_response(meta_cloud_id) + meta_data = meta_response.json() + meta_data['ID'] = meta_id + meta_data['cloud_id'] = cloud_id + meta_data['meta_cloud_id'] = meta_cloud_id + self._collection.add(meta_data) + + self._collection.dump() + + return self._collection def get_doc(self, _id: str) -> Optional[DocumentOrFolder]: """Get a meta item by ID @@ -197,23 +284,12 @@ def get_doc(self, _id: str) -> Optional[DocumentOrFolder]: """ log.debug(f"GETTING DOC {_id}") - response = self.request("GET", "/document-storage/json/2/docs", - params={ - "doc": _id, - "withBlob": True - }) - log.debug(response.url) - data_response = response.json() - log.debug(data_response) + data_response = [item for item in self.get_meta_items() if item.ID == _id] if len(data_response) > 0: - if data_response[0]["Type"] == "CollectionType": - return Folder(**data_response[0]) - elif data_response[0]["Type"] == "DocumentType": - return Document(**data_response[0]) + return data_response[0] else: raise DocumentNotFound(f"Could not find document {_id}") - return None def download(self, document: Document) -> ZipDocument: """Download a ZipDocument @@ -237,7 +313,7 @@ def download(self, document: Document) -> ZipDocument: else: raise UnsupportedTypeError( "We expected a document, got {type}" - .format(type=type(doc))) + .format(type=type(doc))) log.debug("BLOB", document.BlobURLGet) r = self.request("GET", document.BlobURLGet, stream=True) return from_request_stream(document.ID, r) @@ -280,7 +356,7 @@ def upload(self, zip_doc: ZipDocument, to: Folder = Folder(ID="")): if response.ok: doc = Document(**zip_doc.metadata) doc.ID = zip_doc.ID - doc.Parent = to.ID + doc.parent = to.ID return self.update_metadata(doc) else: raise ApiError("an error occured while uploading the document.", @@ -297,8 +373,8 @@ def update_metadata(self, docorfolder: DocumentOrFolder): """ req = docorfolder.to_dict() - req["Version"] = self.get_current_version(docorfolder) + 1 - req["ModifiedClient"] = datetime.utcnow().strftime(RFC3339Nano) + req["version"] = self.get_current_version(docorfolder) + 1 + req["lastModified"] = datetime.utcnow().strftime(RFC3339Nano) res = self.request("PUT", "/document-storage/json/2/upload/update-status", body=[req]) @@ -326,16 +402,16 @@ def get_current_version(self, docorfolder: DocumentOrFolder) -> int: return 0 if not d: return 0 - return int(d.Version) + return int(d.version) def _upload_request(self, zip_doc: ZipDocument) -> str: zip_file, req = zip_doc.create_request() - res = self.request("PUT", "/document-storage/json/2/upload/request", - body=[req]) + res = self.request("POST", self.base_url + "/api/v1/signed-urls/uploads", data=None, + body={"http_method": "PUT", "relative_path": zip_doc.ID}) if not res.ok: raise ApiError( - f"upload request failed with status {res.status_code}", - response=res) + f"upload request failed with status {res.status_code}", + response=res) response = res.json() if len(response) > 0: dest = response[0].get("BlobURLPut", None) @@ -366,8 +442,8 @@ def create_folder(self, folder: Folder): body=[req]) if not res.ok: raise ApiError( - f"upload request failed with status {res.status_code}", - response=res) + f"upload request failed with status {res.status_code}", + response=res) response = res.json() if len(response) > 0: dest = response[0].get("BlobURLPut", None) diff --git a/rmapy/collections.py b/rmapy/collections.py index 34f332f..91599ae 100644 --- a/rmapy/collections.py +++ b/rmapy/collections.py @@ -2,6 +2,12 @@ from .folder import Folder from typing import NoReturn, List, Union from .exceptions import FolderNotFound +from .config import config_meta_path +from pathlib import Path + +from yaml import BaseLoader +from yaml import load as yml_load +from yaml import dump as yml_dump DocumentOrFolder = Union[Document, Folder] @@ -21,6 +27,25 @@ def __init__(self, *items: List[DocumentOrFolder]): for i in items: self.items.append(i) + def to_list(self) -> list: + """ Return a list of the collection. """ + return [item.to_dict() for item in self.items] + + def load(self) -> None: + """ Load Collection from storage """ + storage = [] + if Path.exists(config_meta_path): + with open(config_meta_path, 'r') as config_file: + storage = list(yml_load(config_file.read(), Loader=BaseLoader)) + + for item in storage: + self.add(item) + + def dump(self) -> None: + """ Dump collection to storage""" + with open(config_meta_path, 'w') as config_file: + config_file.write(yml_dump(self.to_list())) + def add(self, doc_dict: dict) -> None: """Add an item to the collection. It wraps it in the correct class based on the Type parameter of the @@ -30,13 +55,13 @@ def add(self, doc_dict: dict) -> None: doc_dict: A dict representing a document or folder. """ - if doc_dict.get("Type", None) == "DocumentType": + if doc_dict.get("type", None) == "DocumentType": self.add_document(doc_dict) - elif doc_dict.get("Type", None) == "CollectionType": + elif doc_dict.get("type", None) == "CollectionType": self.add_folder(doc_dict) else: raise TypeError("Unsupported type: {_type}" - .format(_type=doc_dict.get("Type", None))) + .format(_type=doc_dict.get("type", None))) def add_document(self, doc_dict: dict) -> None: """Add a document to the collection @@ -83,9 +108,9 @@ def children(self, folder: Folder = None) -> List[DocumentOrFolder]: """ if folder: - return [i for i in self.items if i.Parent == folder.ID] + return [i for i in self.items if i.parent == folder.ID] else: - return [i for i in self.items if i.Parent == ""] + return [i for i in self.items if i.parent == ""] def __len__(self) -> int: return len(self.items) diff --git a/rmapy/config.py b/rmapy/config.py index 139c13f..aedf679 100644 --- a/rmapy/config.py +++ b/rmapy/config.py @@ -5,10 +5,14 @@ from typing import Dict +config_path = Path.joinpath(Path.home(), ".rmapi") +config_file_path = Path.joinpath(config_path, 'config') +config_cloudstatus_path = Path.joinpath(config_path, 'cloud') +config_meta_path = Path.joinpath(config_path, 'meta') + + def load() -> dict: """Load the .rmapy config file""" - - config_file_path = Path.joinpath(Path.home(), ".rmapi") config: Dict[str, str] = {} if Path.exists(config_file_path): with open(config_file_path, 'r') as config_file: @@ -24,9 +28,7 @@ def dump(config: dict) -> None: config: A dict containing data to dump to the .rmapi config file. """ - - config_file_path = Path.joinpath(Path.home(), ".rmapi") - + Path(config_path).mkdir(parents=True, exist_ok=True) with open(config_file_path, 'w') as config_file: config_file.write(yml_dump(config)) diff --git a/rmapy/const.py b/rmapy/const.py index edfe403..70aba3b 100644 --- a/rmapy/const.py +++ b/rmapy/const.py @@ -1,7 +1,6 @@ RFC3339Nano = "%Y-%m-%dT%H:%M:%SZ" USER_AGENT = "rmapy" AUTH_BASE_URL = "https://webapp-production-dot-remarkable-production.appspot.com" -BASE_URL = "https://document-storage-production-dot-remarkable-production.appspot.com" # noqa DEVICE_TOKEN_URL = AUTH_BASE_URL + "/token/json/2/device/new" USER_TOKEN_URL = AUTH_BASE_URL + "/token/json/2/user/new" DEVICE = "desktop-windows" diff --git a/rmapy/document.py b/rmapy/document.py index dc0c203..b3b7d1f 100644 --- a/rmapy/document.py +++ b/rmapy/document.py @@ -87,17 +87,17 @@ class Document(Meta): ModifiedClient: When the last change was by the client. Type: Currently there are only 2 known types: DocumentType & CollectionType. - VissibleName: The human name of the object. + visibleName: The human name of the object. CurrentPage: The current selected page of the object. Bookmarked: If the object is bookmarked. - Parent: If empty, this object is is the root folder. This can be an ID + parent: If empty, this object is is the root folder. This can be an ID of a CollectionType. """ def __init__(self, **kwargs): super(Document, self).__init__(**kwargs) - self.Type = "DocumentType" + self.type = "DocumentType" def __str__(self): """String representation of this object""" @@ -206,7 +206,7 @@ def __init__(self, _id=None, doc=None, file=None): "synced": True, "type": "DocumentType", "version": 1, - "VissibleName": "New Document" + "visibleName": "New Document" } self.pagedata = "b''" @@ -241,7 +241,7 @@ def __init__(self, _id=None, doc=None, file=None): with open(doc, 'rb') as fb: self.rm.append(RmPage(page=BytesIO(fb.read()))) name = os.path.splitext(os.path.basename(doc))[0] - self.metadata["VissibleName"] = name + self.metadata["visibleName"] = name if file: self.load(file) @@ -257,7 +257,7 @@ def __repr__(self) -> str: def create_request(self) -> Tuple[BytesIO, dict]: return self.zipfile, { "ID": self.ID, - "Type": "DocumentType", + "type": "DocumentType", "Version": self.metadata["version"] } diff --git a/rmapy/folder.py b/rmapy/folder.py index c5681ca..cd60964 100644 --- a/rmapy/folder.py +++ b/rmapy/folder.py @@ -42,9 +42,9 @@ def __init__(self, name: Optional[str] = None, **kwargs) -> None: """ super(Folder, self).__init__(**kwargs) - self.Type = "CollectionType" + self.type = "CollectionType" if name: - self.VissibleName = name + self.visibleName = name if not self.ID: self.ID = str(uuid4()) @@ -57,7 +57,7 @@ def create_request(self) -> Tuple[BytesIO, dict]: return ZipFolder(self.ID).file, { "ID": self.ID, - "Type": "CollectionType", + "type": "CollectionType", "Version": 1 } diff --git a/rmapy/meta.py b/rmapy/meta.py index a7f2675..def948e 100644 --- a/rmapy/meta.py +++ b/rmapy/meta.py @@ -15,28 +15,27 @@ class Meta(object): ModifiedClient: When the last change was by the client. Type: Currently there are only 2 known types: DocumentType & CollectionType. - VissibleName: The human name of the object. + visibleName: The human name of the object. CurrentPage: The current selected page of the object. Bookmarked: If the object is bookmarked. - Parent: If empty, this object is is the root folder. This can be an ID + parent: If empty, this object is is the root folder. This can be an ID of a CollectionType. """ - - ID = "" - Version = 0 - Message = "" - Success = True - BlobURLGet = "" - BlobURLGetExpires = "" - BlobURLPut = "" - BlobURLPutExpires = "" - ModifiedClient = "" - Type = "" - VissibleName = "" - CurrentPage = 1 - Bookmarked = False - Parent = "" + deleted = False + lastModified = '' + lastOpenedPage = 0 + metadatamodified = False + modified = False + parent = '' + pinned = False + synced = False + type = 'DocumentType' + version = 0, + visibleName = '' + ID = '' + cloud_id = '' + meta_cloud_id = '' def __init__(self, **kwargs): k_keys = self.to_dict().keys() @@ -52,20 +51,17 @@ def to_dict(self) -> dict: a dict of the current object. """ - return { - "ID": self.ID, - "Version": self.Version, - "Message": self.Message, - "Success": self.Success, - "BlobURLGet": self.BlobURLGet, - "BlobURLGetExpires": self.BlobURLGetExpires, - "BlobURLPut": self.BlobURLPut, - "BlobURLPutExpires": self.BlobURLPutExpires, - "ModifiedClient": self.ModifiedClient, - "Type": self.Type, - "VissibleName": self.VissibleName, - "CurrentPage": self.CurrentPage, - "Bookmarked": self.Bookmarked, - "Parent": self.Parent - } - + return dict(deleted=self.deleted, + lastModified=self.lastModified, + lastOpenedPage=self.lastOpenedPage, + metadatamodified=self.metadatamodified, + modified=self.modified, + parent=self.parent, + pinned=self.pinned, + synced=self.synced, + type=self.type, + version=self.version, + visibleName=self.visibleName, + ID=self.ID, + cloud_id=self.cloud_id, + meta_cloud_id = self.meta_cloud_id)