Merge remote-tracking branch 'origin/main' into fix-full-replication-id

Author: rkistner

.changeset/fuzzy-knives-brush.md

@@ -0,0 +1,5 @@
+---
+'@powersync/lib-services-framework': minor
+---
+
+Initial release for service utillities

.changeset/shaggy-parents-join.md

@@ -0,0 +1,5 @@
+---
+'@powersync/service-core': minor
+---
+
+Removed dependency for restricted packages

.env.template

@@ -0,0 +1,3 @@
+# Connections for tests
+MONGO_TEST_UR="mongodb://localhost:27017/powersync_test"
+PG_TEST_URL="postgres://postgres:postgres@localhost:5432/powersync_test"

.github/workflows/development_image_release.yaml

@@ -41,12 +41,6 @@ jobs:
           version: 9
           run_install: false
 
-      - name: Add NPM auth
-        run: |
-          echo //npm.pkg.github.com/:_authToken=\${{secrets.RESTRICTED_PACKAGES_TOKEN}} >> ~/.npmrc
-          echo "//npm.pkg.github.com/journeyapps-platform/:_authToken=${{secrets.RESTRICTED_PACKAGES_TOKEN}}" >> ~/.npmrc
-          echo "@journeyapps-platform:registry=https://npm.pkg.github.com/journeyapps-platform/" >> ~/.npmrc
-
       - name: Get pnpm store directory
         shell: bash
         run: |
@@ -86,6 +80,3 @@ jobs:
           tags: ${{vars.DOCKER_REGISTRY}}:${{steps.get_version.outputs.SERVICE_VERSION}}
           push: true
           file: ./service/Dockerfile
-          # TODO remove this when removing Journey Micro
-          build-args: |
-            GITHUB_TOKEN=${{secrets.RESTRICTED_PACKAGES_TOKEN}}

.github/workflows/development_packages_release.yaml

@@ -27,13 +27,6 @@ jobs:
           version: 9
           run_install: false
 
-      - name: Add NPM auth
-        run: |
-          echo "//registry.npmjs.org/:_authToken=${{secrets.NPM_TOKEN}}" >> ~/.npmrc
-          echo //npm.pkg.github.com/:_authToken=\${{secrets.RESTRICTED_PACKAGES_TOKEN}} >> ~/.npmrc
-          echo "//npm.pkg.github.com/journeyapps-platform/:_authToken=${{secrets.RESTRICTED_PACKAGES_TOKEN}}" >> ~/.npmrc
-          echo "@journeyapps-platform:registry=https://npm.pkg.github.com/journeyapps-platform/" >> ~/.npmrc
-
       - name: Get pnpm store directory
         shell: bash
         run: |
@@ -47,6 +40,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pnpm-store-
 
+      - name: Add NPM auth
+        run: |
+          echo "//registry.npmjs.org/:_authToken=${{secrets.NPM_TOKEN}}" >> ~/.npmrc
+
       - name: Install dependencies
         run: pnpm install
 

.github/workflows/image_release.yaml

@@ -2,60 +2,57 @@
 name: Docker Image Release
 
 on: workflow_dispatch
-  
+
 concurrency: ${{ github.workflow }}-${{ github.ref }}
 
 jobs:
-    release-docker-image:
-        name: Build and Release powersync-service Docker Image
-        runs-on: ubuntu-latest
-        if: github.ref == 'refs/heads/main' 
-        steps:
-          - name: Checkout
-            uses: actions/checkout@v4
-            with:
-              # check out full history
-              # Temporarily needed for changesets
-              fetch-depth: 0
-    
-          - name: Set up QEMU
-            uses: docker/setup-qemu-action@v3
-    
-          - name: Set up Docker Buildx
-            uses: docker/setup-buildx-action@v3
-    
-          - name: Login to Docker Hub
-            uses: docker/login-action@v3
-            with:
-              username: ${{ secrets.DOCKERHUB_USERNAME }}
-              password: ${{ secrets.DOCKERHUB_TOKEN }}
-    
-            # This uses the service's package.json version for the Docker Image tag
-          - name: Get Service Version from package.json
-            id: get_version
-            run: echo "SERVICE_VERSION=$(node -p "require('./service/package.json').version")" >> $GITHUB_OUTPUT
-    
-          - name: Build Image and Push
-            uses: docker/build-push-action@v5
-            with:
-              platforms: linux/arm64,linux/amd64
-              cache-from: type=registry,ref=${{vars.DOCKER_REGISTRY}}:latest
-              context: .
-              tags: ${{vars.DOCKER_REGISTRY}}:latest,${{vars.DOCKER_REGISTRY}}:${{steps.get_version.outputs.SERVICE_VERSION}}
-              push: true
-              file: ./service/Dockerfile
-              # TODO remove this when removing Journey Micro
-              build-args: |
-                GITHUB_TOKEN=${{secrets.RESTRICTED_PACKAGES_TOKEN}}
-    
-          #   # Updates the README section on the DockerHub page
-          - name: Update repo description
-            #  Note that this 3rd party extention is recommended in the DockerHub docs:
-            #  https://docs.docker.com/build/ci/github-actions/update-dockerhub-desc/
-            uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae # v4.0.0
-            with:
-              username: ${{ secrets.DOCKERHUB_USERNAME }}
-              password: ${{ secrets.DOCKERHUB_TOKEN }}
-              repository: ${{vars.DOCKER_REGISTRY}}
-              #   This is the contents of what will be shown on DockerHub
-              readme-filepath: ./service/README.md
+  release-docker-image:
+    name: Build and Release powersync-service Docker Image
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # check out full history
+          # Temporarily needed for changesets
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+        # This uses the service's package.json version for the Docker Image tag
+      - name: Get Service Version from package.json
+        id: get_version
+        run: echo "SERVICE_VERSION=$(node -p "require('./service/package.json').version")" >> $GITHUB_OUTPUT
+
+      - name: Build Image and Push
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/arm64,linux/amd64
+          cache-from: type=registry,ref=${{vars.DOCKER_REGISTRY}}:latest
+          context: .
+          tags: ${{vars.DOCKER_REGISTRY}}:latest,${{vars.DOCKER_REGISTRY}}:${{steps.get_version.outputs.SERVICE_VERSION}}
+          push: true
+          file: ./service/Dockerfile
+
+      #   # Updates the README section on the DockerHub page
+      - name: Update repo description
+        #  Note that this 3rd party extention is recommended in the DockerHub docs:
+        #  https://docs.docker.com/build/ci/github-actions/update-dockerhub-desc/
+        uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae # v4.0.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          repository: ${{vars.DOCKER_REGISTRY}}
+          #   This is the contents of what will be shown on DockerHub
+          readme-filepath: ./service/README.md

.github/workflows/packages_release.yaml

@@ -39,12 +39,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pnpm-store-
 
-      # TODO remove this when no longer needed
-      - name: Temporary Package Credentials
+      - name: Add NPM auth
         run: |
-          echo //npm.pkg.github.com/:_authToken=\${{secrets.RESTRICTED_PACKAGES_TOKEN}} >> ~/.npmrc
-          echo "//npm.pkg.github.com/journeyapps-platform/:_authToken=${{secrets.RESTRICTED_PACKAGES_TOKEN}}" >> ~/.npmrc
-          echo "@journeyapps-platform:registry=https://npm.pkg.github.com/journeyapps-platform/" >> ~/.npmrc
+          echo "//registry.npmjs.org/:_authToken=${{secrets.NPM_TOKEN}}" >> ~/.npmrc
 
       - name: Install dependencies
         run: pnpm install

.github/workflows/test.yml

@@ -70,13 +70,6 @@ jobs:
         with:
           node-version-file: '.nvmrc'
 
-      # TODO remove this when no longer needed
-      - name: Temporary Package Credentials
-        run: |
-          echo //npm.pkg.github.com/:_authToken=\${{secrets.RESTRICTED_PACKAGES_TOKEN}} >> ~/.npmrc
-          echo "//npm.pkg.github.com/journeyapps-platform/:_authToken=${{secrets.RESTRICTED_PACKAGES_TOKEN}}" >> ~/.npmrc
-          echo "@journeyapps-platform:registry=https://npm.pkg.github.com/journeyapps-platform/" >> ~/.npmrc
-
       - uses: pnpm/action-setup@v2
         name: Install pnpm
         with:

.gitignore

@@ -15,6 +15,8 @@ npm-error.log
 .pnpm-debug.log
 .local-dev
 .probes
+powersync.yaml
+sync-rules.yaml
 
 
 packages/*/manifest.json

DEVELOP.md

@@ -14,6 +14,69 @@ The project uses TypeScript. Build packages with
 pnpm build
 ```
 
+# Running Service
+
+## Dependent Services
+
+The PowerSync service requires Postgres and MongoDB server connections. These configuration details can be specified in a `powersync.yaml` (or JSON) configuration file.
+
+See the [Self hosting demo](https://github.com/powersync-ja/self-host-demo) for examples of starting these services.
+
+A quick method for running all required services with a handy backend and frontend is to run the following in a checked-out `self-host-demo` folder.
+
+```bash
+docker compose up --scale powersync=0
+```
+
+Note: The `mongo` hostname specified in the MongoDB replica set needs to be accessible by your host machine if using the Mongo service above.
+
+This will start all the services defined in the Self hosting demo except for the PowerSync service - which will be started from this repository.
+
+## Local Configuration
+
+The `./service` folder contains a NodeJS project which starts all PowerSync service operations.
+
+Copy the template configuration files and configure any changes to your local needs.
+
+```bash
+cd ./service
+cp local-dev/powersync-template.yaml powersync.yaml
+cp local-dev/sync-rules-template.yaml sync-rules.yaml
+```
+
+## Starting Service
+
+The service can be started with watching changes to any consumed source files by running the `pnpm watch:service` command in the repository root.
+
+# Running Tests
+
+Most packages should contain a `test` script which can be executed with `pnpm test`. Some packages may require additional setup to run tests.
+
+## Service Core
+
+Some tests for these packages require a connection to MongoDB and Postgres. Connection strings for these services should be available as environment variables. See [Running Tests](#running-services) for details on configuring those services.
+
+These can be set in a terminal/shell
+
+```bash
+export MONGO_TEST_UR="mongodb://localhost:27017/powersync_test"
+export PG_TEST_URL="postgres://postgres:postgres@localhost:5432/powersync_test"
+```
+
+or by copying the `.env.template` file and using a loader such as [Direnv](https://direnv.net/)
+
+```bash
+cp .env.template .env
+```
+
+## Postgres Configuration
+
+The default `PG_TEST_URL` points to a `powersync_test` database. Ensure this is created by executing the following SQL on your connection.
+
+```SQL
+CREATE DATABASE powersync_test;
+```
+
 # Releases
 
 This repository uses Changesets. Add changesets to changed packages before merging PRs.
@@ -22,4 +85,6 @@ This repository uses Changesets. Add changesets to changed packages before mergi
 changeset add
 ```
 
-Merging a PR with changeset files will automatically create a release PR. Merging the release PR will bump versions, tag and publish packages and the Docker image. The Docker image version is extracted from the `./service/package.json` `version` field.
+Merging a PR with changeset files will automatically create a release PR. Merging the release PR will bump versions, tag and publish packages.
+
+The Docker image is published by manually triggering the `Docker Image Release` Github Action. The Docker image version is extracted from the `./service/package.json` `version` field.