Initial commit

.dockerignore

@@ -0,0 +1 @@
+.venv

.gitignore

@@ -0,0 +1,6 @@
+.venv/
+__pycache__
+.DS_STORE
+scratch
+design/local-only
+.env

Dockerfile

@@ -0,0 +1,45 @@
+FROM ubuntu:bionic-20200311
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y \
+  acl \
+  apt-transport-https \
+  ca-certificates \
+  curl \
+  git \
+  gawk \
+  gnupg-agent \
+  jq \
+  lsof \
+  make \
+  net-tools \
+  rsync \
+  software-properties-common \
+  sshfs \
+  tzdata \
+  unzip \
+  wget \
+  xz-utils \
+  zstd \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install miniconda directly into /opt/miniconda3.
+RUN curl -sL "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" -o /tmp/miniconda.sh &&\
+    chmod +x /tmp/miniconda.sh &&\
+    /tmp/miniconda.sh -b -p /opt/miniconda3 &&\
+    rm /tmp/miniconda.sh &&\
+    chmod -R 777 /opt/miniconda3/ &&\
+    setfacl -d -m u::rwx /opt/miniconda3/ &&\
+    setfacl -d -m g::rwx /opt/miniconda3/ &&\
+    setfacl -d -m o::rwx /opt/miniconda3/
+
+ENV PATH="/opt/miniconda3/bin:/root/.ozy/bin:${PATH}"
+
+ADD environment.yml .
+RUN conda env update --name base --file environment.yml
+
+COPY . /opt/tellus
+WORKDIR /opt/tellus
+ENV PYTHONPATH /opt/tellus
+
+CMD ["/opt/miniconda3/bin/python3", "tellus"]

LICENSE.txt

@@ -0,0 +1,7 @@
+Copyright (c) 2022 Aquatic Technologies LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Makefile

@@ -0,0 +1,114 @@
+SHELL := $(shell which bash) -o pipefail
+MINICONDA := $(HOME)/miniconda3
+CONDA := $(shell which conda || echo $(MINICONDA)/bin/conda)
+VENV := $(PWD)/.venv
+PROJECT_NAME=$(shell basename $(PWD))
+DEPS := $(VENV)/.deps
+DOCKER := $(shell which docker || echo ".docker_is_missing")
+DOCKER_IMAGE = github.com/aquanauts/$(PROJECT_NAME)
+PYTHON := $(VENV)/bin/python
+PYTHON_CMD := PYTHONPATH=$(shell pwd) $(PYTHON)
+PYLINT_CMD := $(PYTHON_CMD) -m pylint $(PROJECT_NAME) test
+COVERAGE_CMD := $(VENV)/bin/coverage
+COVERAGE_REPORTS := $(PWD)/web/public/tests/coverage
+GIT_VERSION := $(shell git rev-list --count HEAD)
+REVISION := $(GIT_VERSION)
+TELLUS_VERSION := $(REVISION)
+TELLUS_PERSISTENCE_DIR := '/tmp/tellus'
+TELLUS_CMD := $(PYTHON_CMD) tellus --persistence-root=$(TELLUS_PERSISTENCE_DIR)
+
+ifndef VERBOSE
+.SILENT:
+endif
+
+.phony: help
+help:
+	grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+$(CONDA):
+	echo "Installing Miniconda3 to $(MINICONDA)"
+	wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/miniconda.sh
+	bash /tmp/miniconda.sh -u -b -p "$(HOME)/miniconda3"
+	rm /tmp/miniconda.sh
+	$(CONDA) update -y conda
+
+$(PYTHON): | $(CONDA)
+	$(CONDA) env create -p $(VENV)
+
+$(DEPS): environment.yml $(PYTHON)
+	$(CONDA) env update -p $(VENV) -f environment.yml
+	cp environment.yml $(DEPS)
+
+clean:
+	find . -name __pycache__ | xargs rm -rf
+	rm -rf $(VENV)
+
+.phony: test
+test: $(DEPS) todo tasks ## Run linting, todos, unit, and integration tests
+	#$(PYLINT_CMD)
+	# Ignores smoketests, see below
+	$(PYTHON_CMD) -m pytest --ignore test/smoketests
+
+.phony: coverage
+coverage: $(DEPS) todo tasks ## Run tests with coverage.py
+	$(COVERAGE_CMD) run -m pytest --ignore test/smoketests; $(COVERAGE_CMD) report
+
+.phony: cov-html
+cov-html: $(DEPS) todo tasks ## Run tests with coverage.py and dump the html report into Tellus (yes, cheating)
+	$(COVERAGE_CMD) run -m pytest --ignore test/smoketests; $(COVERAGE_CMD) html --directory=$(COVERAGE_REPORTS)
+
+smoketest: $(DEPS) ## Run the smoketests (which have environmental dependencies)
+	$(PYTHON_CMD) -m pytest test/smoketests
+
+watch: $(DEPS) ## Run unit tests and lint continuously
+	$(PYTHON_CMD) -m pytest_watch --runner $(VENV)/bin/pytest -n --onpass '$(PYLINT_CMD)' --ignore $(VENV) --ignore test/smoketests
+
+run: $(DEPS)  ## run a local server - will try to persist in $(TELLUS_PERSISTENCE_DIR)
+	$(TELLUS_CMD)
+
+rundebug: $(DEPS)  ## run a local server in debug mode - very chatty
+	$(TELLUS_CMD) --debug
+
+repl: $(DEPS) ## Runs a REPL
+	$(VENV)/bin/ipython
+
+todo: ## Find technical debt
+	-(find tellus test web/public -name '*.py' -o -name '*.js' | grep -Ev '(web/public/tests/lib/jasmine|web/public/vendor/moment)' | xargs grep -i --color TODO)
+
+tasks: ## Find tasks (comments for intended future work)
+	-(find tellus test web/public/js -name '*.py' -o -name '*.js' | xargs grep -i --color 'tellus-task')
+
+.phony: git-version
+git-version: ## Print the current git revision of Tellus
+	echo $(GIT_VERSION)
+
+.phony: version
+version: ## Print the current full version of Tellus
+	echo "Overall version:  $(TELLUS_VERSION)"
+	echo "version contains:"
+	cat tellus/version.py
+
+update-version: ## update the version in version.py - run in various places in the Makefile to ensure up to date
+	echo "__version__ = '$(TELLUS_VERSION)'" > tellus/version.py
+	echo "version.py:"
+	cat tellus/version.py
+
+docker: $(DOCKER) ## build our docker image
+	$(DOCKER) build . -t $(DOCKER_IMAGE):latest
+
+docker-run: $(DEPS) $(DOCKER) docker ## run locally in Docker
+	docker run -it -p 8080:8080/tcp $(DOCKER_IMAGE)
+
+release: update-version docker ## Push a release to artifactory
+	docker push $(DOCKER_IMAGE):latest
+
+shell: docker fastshell ## Open a new shell inside Docker container for development
+
+fastshell:  ## Open a new shell WITHOUT building docker - here be dragons
+	docker run -v $(CURDIR):/src/$(PROJECT_NAME) -w /src/$(PROJECT_NAME) --entrypoint /bin/bash -it $(DOCKER_IMAGE)
+
+runshell:  ## Open a new shell WITHOUT building docker (see fastshell), grabbing port 8080 for running locally
+	docker run -v $(CURDIR):/src/$(PROJECT_NAME) -w /src/$(PROJECT_NAME) -p 8080:8080/tcp --entrypoint /bin/bash -it $(DOCKER_IMAGE)
+
+blacken:  ## blacken code
+	black ./

README.md

@@ -0,0 +1,42 @@
+# Tellus
+Tellus is a central hub for connecting various information
+
+Tellus' basic unit of information is a Tell. Tells have a bunch of info including tags, which are used to tie different Tells together.
+
+The current sources of information within Tellus are listed below.
+
+## Go
+Tellus' primary function is to allow us to define short, human-readable aliases for useful URLs within Aqauatic.
+
+## User (aka 'Be')
+Tellus has a pseudo-login that will let you "Become" a user (ideally, yourself).  Right now this is just used for audit info but Tellus will do more with this eventually.
+
+Note that it is piggybacking off of github users for valid user names for now.
+
+## Other Data Sources
+Tellus draws from a number of different data sources to populate itself and tie things together.
+
+### Github - tellus.yml files
+If you put a [tellus.yml](https://github.com/aquanauts/tellus/blob/master/tellus.yml) file into Github, Tellus will automatically grab it and attempt to create or update Tells within Tellus based on the information inside.  The only required attribute is 'alias'.
+
+The first item in the tellus.yml is considered the "primary" Tell, and other Tells can be related to that one.
+
+Note: you can also add arbitrary attributes to these files and Tellus will put them into an 'Additional Data' block (see below).
+
+## Tells
+
+Tells consist of the following information:
+
+- `alias` (*required and **opinionated** *): the key of the Tell.
+	* Aliases *must be* all lowercase, separated by dashes.  Tellus will largely try to convert an alias to this format.
+	* At present, Aliases are immutable.
+- `go-url`: the url for the Tell, to which Go will redirect.
+	* This is not required.
+	* In certain cases, this will be automatically generated
+- `description`: additional human-readable information about the Tell.
+- `tags`:  a set of identifying tags for the Tell.  These affect display and link Tells together in various locations within Tellus.
+- `category` (read-only): Tellus' internal categories for the Tell (used for certain behaviors).
+
+### Additional Data
+
+Tells coming from other sources will sometimes have additional information attached to them.   Tellus will attempt to store and display those values as best it can on the Tell.

design/design.txt

@@ -0,0 +1,103 @@
+This is design notes and to-dos for now
+
+CURRENT MAIN TASKS
+    - User Page
+        - Remember to come back and deal with downloading of avatar images (going to have to cache these locally)
+        - Why is the github URL getting populated when it doesn't exist?  (e.g., Ellen)
+    - Sheet Consumption
+    - Search
+        - Need to handle single character searches right (check for disallowed strings)
+    - TellusSource - Tellus' internal source to do its own setup functions
+        - Ensure there are some basic Tells for things like 'go' etc.
+    - Coverage - /tests/coverage/index.html
+
+
+2020-10-10:
+    In massive refactor, keeping some notes:
+        - Need to update existing Tells based on Categories in Migration
+        - Figure out last_updated bug (and maybe just put the info into the data dict) - think this was just the user issue
+
+DNS
+    - Check port 80 instead (in addition?) on load
+    - Only reload every once in a while
+    - Note last load, allow force?
+
+GO
+    -
+
+TELLUS general
+    NEXT
+        - Fix DNS so that it removes unreachable DNS entries after some period of time
+        - Consider whether to collapse Teller persistence into a parameter...
+        - Create something like data migrations...and save file versioning
+            - SAVE FILE VERSIONING
+                - Migrations from prev file to this one?
+
+        THINK ABOUT:  standard "value" items (e.g., alias) are easier to fill in templates
+            'look for tellus-alias' anywhere
+        Move other config into Vault
+            e.g., Github key
+            Create a configuration class
+    ISSUES
+        * Edit -> Delete is bad UI...add the delete/really? thing
+        * TELLUS SHOULD NOT OVERWRITE DATA, EVER (currently sources will do so)
+            x Source load should only be additive
+            ? Maybe always put source values in source map?
+            x Add source of "human" -> treats it as additive (Edit button?)
+                ? Collapse TELLUS_HUMAN -> TELLUS_GO?
+        - Fix queries/tellus category stuff (fully qualify categories)
+    - Sources
+        - Arbitrary adding of different types of sources
+            - Confluence
+            - Sheet (needs a bunch of config - to do sometime later)
+        - aiohttp multiple threads?  Asyncify some of the lookups?
+        - Prioritize updates of Tell data?  Right now it's basically last one wins...
+        - Add "purge/reload" for non-human touched items
+    Deployment
+        - Have Tellus' deployment link to its logs?
+    Tellicious
+        - what kind of a tell is this?
+            - no alias
+            -
+        - javascript
+        - rss -> Slack?
+    Feature Requests
+        tellus tiles a la Jumpcloud
+        CNAme -> A record mapping
+    Tells
+        - allow space delimited tags
+        - Make persistence safer (crazy git idea, or some other versioning something something)
+        - Look at creation persistence - I think happening outside of the Teller
+            Maybe did this because DNS...think about timing
+    Clean up the UI
+        ? Auto-gen Tellus display page attributes from dict
+        - Make prettier?  (use bootstrap theme?)
+        - figure out Bootstrap validation
+    Github Source
+        - Add all repos
+        - Give activity for the repo (sparkline of commits, people)
+        - Edges
+    Configuration
+        - Move all of the stuff out to real config files
+        - Need config file
+    Slackbot
+        - whatis (or just tellus?)
+        - whois
+    S/M
+        - History?
+
+
+- Validation
+
+Features I'd like to add:
+    - Magic tags (e.g., "Dashboard")
+
+Replicate for Go:
+javascript:q=location.href;if(document.getSelection){d=document.getSelection();}else{d='';};p=document.title;void(open('https://pinboard.in/add?url='+encodeURIComponent(q)+'&description='+encodeURIComponent(d)+'&title='+encodeURIComponent(p),'Pinboard','toolbar=no,width=700,height=350'));
+javascript:q=location.href;if(document.getSelection){d=document.getSelection();}else{d='';};p=document.title;void(open('https://pinboard.in/add?url='+encodeURIComponent(q)+'&description='+encodeURIComponent(d)+'&title='+encodeURIComponent(p),'Pinboard','toolbar=no,width=700,height=350'));
+
+OLD NOTES
+---------
+    # This is now handled a little more simply, but useful reminder...
+    # Setting a debug email in the browser looks like this:
+    # tellususer = "{\"created\": 1602255557\054 \"session\": {\"tellus_debug_email\": \"[email protected]\"\054 \"tellususer\": \"user\"}}"

environment.yml

@@ -0,0 +1,37 @@
+---
+name: tellus
+
+channels:
+  - conda-forge
+  - defaults
+
+dependencies:
+  - ipython
+  - pylint
+  - humanfriendly
+  - aiohttp
+  - aiohttp-session
+  - aiohttp-security
+  - trafaret
+  - pytest-mock
+  - pytest-aiohttp
+  - pyfakefs
+  - sortedcontainers
+  - pygithub
+  - pyyaml
+  - requests-mock
+  - jsonpickle
+  - pip
+  - oauth2client
+  - gspread
+  - google-api-python-client
+  - asynctest
+  - python-dateutil
+  - black
+  - atlassian-python-api
+  - pytest-watch
+  - python-dotenv
+  - pytest-asyncio
+  - pandas
+  - fuzzywuzzy
+  - coverage

pylintrc

@@ -0,0 +1,4 @@
+[MASTER]
+
+# Only include warnings and errors
+disable=C,R

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+filterwarnings =
+    ignore:.*Using or importing the ABCs.*:DeprecationWarning

tellus.yml

@@ -0,0 +1,18 @@
+# Tellus' tool file
+# Note that the Tellus yml file has some special tells for configuring Tellus itself
+alias: tellus
+description:  Tellus is our central portal/hub, living at github.com/aquanauts/tellus - you are, probably, here.
+go_url: https://github.com/users
+tags:  tellus, tools
+colors: yellow and purple
+about: https://github.com/users/tellusstaticfiles/tellus.html
+builds: https://build.github.com/users/#/builders?tags=%2Btellus
+deploy:  https://deploy.github.com/users/ui/jobs/tellus
+# tools:  all
+---
+alias: tellus-config-tools
+description:  A configuration Tell for Tellus - configures certain "magic word" URLs for the tellus.yml files.
+tags: tellus, tools, tellus-config
+builds: https://build.github.com/users/#/builders/<ID>
+deploy: https://deploy.github.com/users/ui/jobs/<alias>
+github-repo: https://github.com/users/<org>/<alias>

tellus/__init__.py

@@ -0,0 +1 @@
+from .version import __version__

tellus/__main__.py

@@ -0,0 +1,4 @@
+import sys
+from tellus.main import main
+
+main(sys.argv)