From 9826a39955d4c37683afbeb9684bfeb700107a62 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Sat, 22 Oct 2022 16:21:50 +0200
Subject: [PATCH 01/19] feat: Cockroach DB support
---
.github/workflows/test.yml | 3 +-
backend/app/build.gradle | 21 ++++-
.../test/resources/application-cockroach.yaml | 7 ++
.../configuration/LiquibaseConfiguration.kt | 2 +-
.../activity/ActivityRevisionRepository.kt | 2 +-
.../main/resources/db/changelog/schema.xml | 89 +++++++++++++++++++
.../kotlin/io/tolgee/CleanDbTestListener.kt | 12 ++-
.../testing/AbstractTransactionalTest.kt | 2 -
8 files changed, 131 insertions(+), 7 deletions(-)
create mode 100644 backend/app/src/test/resources/application-cockroach.yaml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index fb0c8e98cf..078cfdb8d2 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -58,6 +58,7 @@ jobs:
fail-fast: false
matrix:
command: [ "server-app:runContextRecreatingTests", "server-app:runStandardTests", "data:test" ]
+ profile: [ "default, cockroach" ]
steps:
- uses: actions/checkout@v2
@@ -91,7 +92,7 @@ jobs:
tar -xzf ~/backend-testing.tgz ./backend/testing/build
- name: Run backend tests
- run: ./gradlew ${{ matrix.command }}
+ run: SPRING_PROFILES_ACTIVE=${{ matrix.profile }} ./gradlew ${{ matrix.command }}
env:
SKIP_SERVER_BUILD: true
diff --git a/backend/app/build.gradle b/backend/app/build.gradle
index b5ffe3b626..5ce9c1fbc6 100644
--- a/backend/app/build.gradle
+++ b/backend/app/build.gradle
@@ -167,19 +167,38 @@ dependencies {
test {
useJUnitPlatform()
maxHeapSize = "2048m"
- //maxParallelForks = (int) (Runtime.runtime.availableProcessors() / 2 + 1)
+}
+
+task testCockroach(type: Test, group: 'verification') {
+ useJUnitPlatform()
+ maxHeapSize = "2048m"
+ systemProperty 'SPRING_PROFILES_ACTIVE', 'cockroach'
}
task runContextRecreatingTests(type: Test, group: 'verification') {
+ dependsOn "startCockroachDb"
useJUnitPlatform {
includeTags "contextRecreating"
}
+ finalizedBy "stopCockroachDb"
}
task runStandardTests(type: Test, group: 'verification') {
+ dependsOn "startCockroachDb"
useJUnitPlatform {
excludeTags "contextRecreating"
}
+ finalizedBy "stopCockroachDb"
+}
+
+task startCockroachDb(type: Exec){
+ onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
+ commandLine "bash", "-c", "docker run --rm -d --name=roach -p 8086:8080 -p 26257:26257 cockroachdb/cockroach:latest start-single-node --insecure"
+}
+
+task stopCockroachDb(type: Exec){
+ onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
+ commandLine "bash", "-c", "docker stop roach"
}
springBoot {
diff --git a/backend/app/src/test/resources/application-cockroach.yaml b/backend/app/src/test/resources/application-cockroach.yaml
new file mode 100644
index 0000000000..6f7e8b7d60
--- /dev/null
+++ b/backend/app/src/test/resources/application-cockroach.yaml
@@ -0,0 +1,7 @@
+spring:
+ datasource:
+ url: jdbc:postgresql://localhost:26257/postgres
+ username: root
+tolgee:
+ postgres-autostart:
+ enabled: false
diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt
index 2bf3bcdf82..e40dc9d813 100644
--- a/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt
@@ -16,7 +16,7 @@ class LiquibaseConfiguration {
liquibase.dataSource = dataSource
liquibase.changeLog = "classpath:db/changelog/schema.xml"
liquibase.defaultSchema = "public"
-
+ liquibase.setChangeLogParameters(mapOf("isCockroach" to "true"))
return liquibase
}
}
diff --git a/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt b/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt
index 4669f24598..c45349b145 100644
--- a/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/repository/activity/ActivityRevisionRepository.kt
@@ -61,7 +61,7 @@ interface ActivityRevisionRepository : JpaRepository<ActivityRevision, Long> {
@Query(
"""
- select count(ar.id) as count, function('to_char', ar.timestamp, 'yyyy-MM-dd') as date
+ select count(ar.id) as count, cast(cast(ar.timestamp as date) as text) as date
from ActivityRevision ar
where ar.projectId = :projectId
group by date
diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml
index 1914365bf3..aa7fdeb8a9 100644
--- a/backend/data/src/main/resources/db/changelog/schema.xml
+++ b/backend/data/src/main/resources/db/changelog/schema.xml
@@ -82,6 +82,44 @@
<column name="created_by_id" type="BIGINT"/>
</createTable>
</changeSet>
+ <changeSet author="jenik (generated)" id="1585252533730-6">
+ <preConditions onFail="CONTINUE">
+ <not>
+ <changeLogPropertyDefined property="isCockroach" value="true"/>
+ </not>
+ </preConditions>
+ <createTable tableName="source">
+ <column autoIncrement="true" name="id" type="BIGINT">
+ <constraints primaryKey="true" primaryKeyName="sourcePK"/>
+ </column>
+ <column name="created_at" type="TIMESTAMP WITHOUT TIME ZONE">
+ <constraints nullable="false"/>
+ </column>
+ <column name="updated_at" type="TIMESTAMP WITHOUT TIME ZONE">
+ <constraints nullable="false"/>
+ </column>
+ <column name="name" type="VARCHAR(255)"/>
+ <column name="repository_id" type="BIGINT"/>
+ </createTable>
+ </changeSet>
+ <changeSet author="jenik (generated)" id="1585252533730-6crdb">
+ <preConditions onFail="CONTINUE">
+ <changeLogPropertyDefined property="isCockroach" value="true"/>
+ </preConditions>
+ <createTable tableName="source">
+ <column autoIncrement="true" name="id" type="BIGINT">
+ <constraints primaryKey="true" primaryKeyName="sourcePK"/>
+ </column>
+ <column name="created_at" type="TIMESTAMP WITHOUT TIME ZONE">
+ <constraints nullable="false"/>
+ </column>
+ <column name="updated_at" type="TIMESTAMP WITHOUT TIME ZONE">
+ <constraints nullable="false"/>
+ </column>
+ <column name="name" type="VARCHAR(2000)"/>
+ <column name="repository_id" type="BIGINT"/>
+ </createTable>
+ </changeSet>
<changeSet author="jenik (generated)" id="1585252533730-6">
<createTable tableName="source">
<column autoIncrement="true" name="id" type="BIGINT">
@@ -444,6 +482,14 @@
<changeSet author="jenik (generated)" id="1621606909199-2">
<createSequence incrementBy="100" sequenceName="hibernate_sequence" startValue="1000000000"/>
</changeSet>
+ <changeSet author="jenik (generated)" id="1621606909199-2crdb">
+ <preConditions onFail="CONTINUE">
+ <changeLogPropertyDefined property="isCockroach" value="true"/>
+ </preConditions>
+ <sql>
+ select setval('hibernate_sequence', 1000000000)
+ </sql>
+ </changeSet>
<changeSet author="jenik (generated)" id="1621606909199-3">
<createTable tableName="import">
<column name="id" type="BIGINT">
@@ -1088,6 +1134,11 @@
</createTable>
</changeSet>
<changeSet author="jenik (generated)" id="1624975131291-13">
+ <preConditions onFail="CONTINUE">
+ <not>
+ <changeLogPropertyDefined property="isCockroach" value="true"/>
+ </not>
+ </preConditions>
<createTable tableName="revision">
<column autoIncrement="true" name="id" type="INT">
<constraints nullable="false" primaryKey="true" primaryKeyName="revisionPK"/>
@@ -1098,6 +1149,20 @@
<column name="author_id" type="BIGINT"/>
</createTable>
</changeSet>
+ <changeSet author="jenik (generated)" id="1624975131291-13crdb">
+ <preConditions onFail="CONTINUE">
+ <changeLogPropertyDefined property="isCockroach" value="true"/>
+ </preConditions>
+ <createTable tableName="revision">
+ <column autoIncrement="true" name="id" type="BIGINT">
+ <constraints nullable="false" primaryKey="true" primaryKeyName="revisionPK"/>
+ </column>
+ <column name="timestamp" type="BIGINT">
+ <constraints nullable="false"/>
+ </column>
+ <column name="author_id" type="BIGINT"/>
+ </createTable>
+ </changeSet>
<changeSet author="jenik (generated)" id="1624975131291-14">
<createTable tableName="screenshot_aud">
<column name="id" type="BIGINT">
@@ -1356,7 +1421,21 @@
<changeSet author="jenik (generated)" id="1626957302292-4">
<addNotNullConstraint columnDataType="bigint" columnName="key_id" tableName="translation" validate="true"/>
</changeSet>
+ <changeSet author="jenik" id="1626957302295-0">
+ <preConditions onFail="CONTINUE">
+ <changeLogPropertyDefined property="isCockroach" value="true"/>
+ </preConditions>
+ <sql>
+ set
+ enable_experimental_alter_column_type_general = true
+ </sql>
+ </changeSet>
<changeSet author="jenik (generated)" id="1626957302295-1">
+ <preConditions onFail="CONTINUE">
+ <not>
+ <changeLogPropertyDefined property="isCockroach" value="true"/>
+ </not>
+ </preConditions>
<modifyDataType tableName="revision" columnName="id" newDataType="BIGINT"/>
</changeSet>
<changeSet author="jenik (generated)" id="1633085681853-3">
@@ -1399,6 +1478,11 @@
constraintName="organization_third_party_billing_id_unique" tableName="organization"/>
</changeSet>
<changeSet author="jenik" id="1637177424380-0">
+ <preConditions onFail="CONTINUE">
+ <not>
+ <changeLogPropertyDefined property="isCockroach" value="true"/>
+ </not>
+ </preConditions>
<modifyDataType
columnName="name"
newDataType="varchar(2000)"
@@ -2101,6 +2185,11 @@
<addUniqueConstraint columnNames="key_hash" constraintName="api_key_hash_unique" tableName="api_key"/>
</changeSet>
<changeSet author="jenik (generated)" id="1661361670848-3">
+ <preConditions onFail="CONTINUE">
+ <not>
+ <changeLogPropertyDefined property="isCockroach" value="true"/>
+ </not>
+ </preConditions>
<sqlFile path="classpath:db/changelog/hideApiKeyMigration.sql" splitStatements="false"/>
</changeSet>
<changeSet author="jenik (generated)" id="1661420385794-1">
diff --git a/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt b/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt
index d25bdc0ad6..eb6e4e5213 100644
--- a/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt
+++ b/backend/testing/src/main/kotlin/io/tolgee/CleanDbTestListener.kt
@@ -5,6 +5,7 @@ import org.slf4j.LoggerFactory
import org.springframework.context.ApplicationContext
import org.springframework.test.context.TestContext
import org.springframework.test.context.TestExecutionListener
+import java.sql.Connection
import java.sql.ResultSet
import javax.sql.DataSource
import kotlin.system.measureTimeMillis
@@ -47,7 +48,7 @@ class CleanDbTestListener : TestExecutionListener {
val ds: DataSource = appContext.getBean(DataSource::class.java)
ds.connection.use { conn ->
val stmt = conn.createStatement()
- val databaseName: Any = "postgres"
+ val databaseName: Any = getDatabaseName(conn)
val ignoredTablesString = ignoredTables.joinToString(", ") { "'$it'" }
val rs: ResultSet = stmt.executeQuery(
String.format(
@@ -71,6 +72,15 @@ class CleanDbTestListener : TestExecutionListener {
}
}
+ private fun getDatabaseName(conn: Connection): String {
+ val rs = conn.getMetaData().catalogs
+ val data = mutableListOf<String>()
+ while (rs.next()) {
+ data.add(rs.getString(1))
+ }
+ return data.single()
+ }
+
@Throws(Exception::class)
override fun afterTestMethod(testContext: TestContext) {
}
diff --git a/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt b/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt
index 75cd912aa6..829223fee5 100644
--- a/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt
+++ b/backend/testing/src/main/kotlin/io/tolgee/testing/AbstractTransactionalTest.kt
@@ -2,7 +2,6 @@ package io.tolgee.testing
import io.tolgee.CleanDbTestListener
import org.springframework.beans.factory.annotation.Autowired
-import org.springframework.test.context.ActiveProfiles
import org.springframework.test.context.TestExecutionListeners
import org.springframework.test.context.support.DependencyInjectionTestExecutionListener
import org.springframework.test.context.transaction.TestTransaction
@@ -16,7 +15,6 @@ import javax.persistence.EntityManager
CleanDbTestListener::class
]
)
-@ActiveProfiles(profiles = ["local"])
abstract class AbstractTransactionalTest {
@Autowired
protected lateinit var entityManager: EntityManager
From 7d6eba2c262de8926954d3c09ea13ba0bccbbe9b Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Sat, 22 Oct 2022 16:31:57 +0200
Subject: [PATCH 02/19] feat: Cockroach DB support > fix workflow
---
.github/workflows/test.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 078cfdb8d2..5ec8a34449 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -58,7 +58,7 @@ jobs:
fail-fast: false
matrix:
command: [ "server-app:runContextRecreatingTests", "server-app:runStandardTests", "data:test" ]
- profile: [ "default, cockroach" ]
+ profile: [ "default", "cockroach" ]
steps:
- uses: actions/checkout@v2
From 2d5f9b5b161965053acd71cdf47d2b5dbb29e76c Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Sat, 22 Oct 2022 16:57:51 +0200
Subject: [PATCH 03/19] feat: Cockroach DB support > cockroach switch
---
backend/app/build.gradle | 2 +-
.../src/test/resources/application-cockroach.yaml | 2 ++
.../tolgee/configuration/LiquibaseConfiguration.kt | 8 ++++++--
.../configuration/tolgee/DatabaseProperties.kt | 12 ++++++++++++
.../tolgee/configuration/tolgee/TolgeeProperties.kt | 3 ++-
.../data/src/main/resources/db/changelog/schema.xml | 9 ---------
6 files changed, 23 insertions(+), 13 deletions(-)
create mode 100644 backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/DatabaseProperties.kt
diff --git a/backend/app/build.gradle b/backend/app/build.gradle
index 5ce9c1fbc6..d19a769b7b 100644
--- a/backend/app/build.gradle
+++ b/backend/app/build.gradle
@@ -193,7 +193,7 @@ task runStandardTests(type: Test, group: 'verification') {
task startCockroachDb(type: Exec){
onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
- commandLine "bash", "-c", "docker run --rm -d --name=roach -p 8086:8080 -p 26257:26257 cockroachdb/cockroach:latest start-single-node --insecure"
+ commandLine "bash", "-c", "docker run --rm -d --name=roach -p 8086:8080 -p 26257:26257 cockroachdb/cockroach:22.1 start-single-node --insecure"
}
task stopCockroachDb(type: Exec){
diff --git a/backend/app/src/test/resources/application-cockroach.yaml b/backend/app/src/test/resources/application-cockroach.yaml
index 6f7e8b7d60..cd41838663 100644
--- a/backend/app/src/test/resources/application-cockroach.yaml
+++ b/backend/app/src/test/resources/application-cockroach.yaml
@@ -3,5 +3,7 @@ spring:
url: jdbc:postgresql://localhost:26257/postgres
username: root
tolgee:
+ database:
+ type: COCKROACH
postgres-autostart:
enabled: false
diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt
index e40dc9d813..6835e58e4f 100644
--- a/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/configuration/LiquibaseConfiguration.kt
@@ -1,5 +1,7 @@
package io.tolgee.configuration
+import io.tolgee.configuration.tolgee.DatabaseProperties
+import io.tolgee.configuration.tolgee.TolgeeProperties
import liquibase.integration.spring.SpringLiquibase
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
@@ -10,13 +12,15 @@ import javax.sql.DataSource
class LiquibaseConfiguration {
@Bean
@Primary
- fun liquibase(dataSource: DataSource): SpringLiquibase {
+ fun liquibase(dataSource: DataSource, properties: TolgeeProperties): SpringLiquibase {
val liquibase = SpringLiquibase()
liquibase.dataSource = dataSource
liquibase.changeLog = "classpath:db/changelog/schema.xml"
liquibase.defaultSchema = "public"
- liquibase.setChangeLogParameters(mapOf("isCockroach" to "true"))
+ if (properties.database.type == DatabaseProperties.DatabaseType.COCKROACH) {
+ liquibase.setChangeLogParameters(mapOf("isCockroach" to "true"))
+ }
return liquibase
}
}
diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/DatabaseProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/DatabaseProperties.kt
new file mode 100644
index 0000000000..87347e1ce7
--- /dev/null
+++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/DatabaseProperties.kt
@@ -0,0 +1,12 @@
+package io.tolgee.configuration.tolgee
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+
+@ConfigurationProperties(prefix = "tolgee.database")
+class DatabaseProperties {
+ var type: DatabaseType = DatabaseType.POSTGRES
+
+ enum class DatabaseType {
+ COCKROACH, POSTGRES
+ }
+}
diff --git a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt
index 8828f21240..2491ce9ae3 100644
--- a/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/configuration/tolgee/TolgeeProperties.kt
@@ -31,5 +31,6 @@ open class TolgeeProperties(
var postgresAutostart: PostgresAutostartProperties = PostgresAutostartProperties(),
var sendInBlueProperties: SendInBlueProperties = SendInBlueProperties(),
open var import: ImportProperties = ImportProperties(),
- var rateLimitProperties: RateLimitProperties = RateLimitProperties()
+ var rateLimitProperties: RateLimitProperties = RateLimitProperties(),
+ var database: DatabaseProperties = DatabaseProperties()
)
diff --git a/backend/data/src/main/resources/db/changelog/schema.xml b/backend/data/src/main/resources/db/changelog/schema.xml
index aa7fdeb8a9..336ce696fb 100644
--- a/backend/data/src/main/resources/db/changelog/schema.xml
+++ b/backend/data/src/main/resources/db/changelog/schema.xml
@@ -1421,15 +1421,6 @@
<changeSet author="jenik (generated)" id="1626957302292-4">
<addNotNullConstraint columnDataType="bigint" columnName="key_id" tableName="translation" validate="true"/>
</changeSet>
- <changeSet author="jenik" id="1626957302295-0">
- <preConditions onFail="CONTINUE">
- <changeLogPropertyDefined property="isCockroach" value="true"/>
- </preConditions>
- <sql>
- set
- enable_experimental_alter_column_type_general = true
- </sql>
- </changeSet>
<changeSet author="jenik (generated)" id="1626957302295-1">
<preConditions onFail="CONTINUE">
<not>
From 728f12f98ab7c004f66607bf2e71da4db05d28c5 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Sat, 22 Oct 2022 17:06:11 +0200
Subject: [PATCH 04/19] feat: Cockroach DB support > image vers
---
backend/app/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/backend/app/build.gradle b/backend/app/build.gradle
index d19a769b7b..87549ba7b4 100644
--- a/backend/app/build.gradle
+++ b/backend/app/build.gradle
@@ -193,7 +193,7 @@ task runStandardTests(type: Test, group: 'verification') {
task startCockroachDb(type: Exec){
onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
- commandLine "bash", "-c", "docker run --rm -d --name=roach -p 8086:8080 -p 26257:26257 cockroachdb/cockroach:22.1 start-single-node --insecure"
+ commandLine "bash", "-c", "docker run --rm -d --name=roach -p 8086:8080 -p 26257:26257 cockroachdb/cockroach:v22.1.9 start-single-node --insecure"
}
task stopCockroachDb(type: Exec){
From 5f638712a95b8dc32f383f5b51417fe519146cd9 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Sat, 22 Oct 2022 18:44:28 +0200
Subject: [PATCH 05/19] feat: Cockroach DB support > e2e with cockroach
---
.github/workflows/test.yml | 2 ++
backend/app/build.gradle | 2 +-
.../main/resources/application-cockroach.yaml | 9 +++++++++
.../testDataBuilder/data/TagsTestData.kt | 6 ++++--
.../query_builders/TranslationsViewBuilder.kt | 16 ++++++++++++----
e2e/docker-compose.yml | 10 +++++++++-
gradle/e2e.gradle | 15 ++++++++++++---
7 files changed, 49 insertions(+), 11 deletions(-)
create mode 100644 backend/app/src/main/resources/application-cockroach.yaml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5ec8a34449..c12a189dde 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -118,6 +118,7 @@ jobs:
matrix:
total_jobs: [ 10 ]
job_index: [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ]
+ profile: ["cockroach", "default"]
steps:
- uses: actions/checkout@v2
@@ -188,6 +189,7 @@ jobs:
SKIP_INSTALL_E2E_DEPS: true
E2E_TOTAL_JOBS: ${{matrix.total_jobs}}
E2E_JOB_INDEX: ${{matrix.job_index}}
+ SPRING_PROFILES_ACTIVE: ${{matrix.profile}}
- uses: actions/upload-artifact@v2
if: failure()
diff --git a/backend/app/build.gradle b/backend/app/build.gradle
index 87549ba7b4..7228642489 100644
--- a/backend/app/build.gradle
+++ b/backend/app/build.gradle
@@ -193,7 +193,7 @@ task runStandardTests(type: Test, group: 'verification') {
task startCockroachDb(type: Exec){
onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
- commandLine "bash", "-c", "docker run --rm -d --name=roach -p 8086:8080 -p 26257:26257 cockroachdb/cockroach:v22.1.9 start-single-node --insecure"
+ commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach:v22.1.9 start-single-node --insecure"
}
task stopCockroachDb(type: Exec){
diff --git a/backend/app/src/main/resources/application-cockroach.yaml b/backend/app/src/main/resources/application-cockroach.yaml
new file mode 100644
index 0000000000..84f1e6ba8f
--- /dev/null
+++ b/backend/app/src/main/resources/application-cockroach.yaml
@@ -0,0 +1,9 @@
+spring:
+ datasource:
+ url: jdbc:postgresql://cockroachdb:26257/postgres
+ username: root
+tolgee:
+ database:
+ type: COCKROACH
+ postgres-autostart:
+ enabled: false
diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt
index 28b6d83e3f..7768ee523e 100644
--- a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt
@@ -55,16 +55,18 @@ class TagsTestData : BaseTestData("tagsTestUser", "tagsTestProject") {
}
}
(1..20).forEach { keyNum ->
+ val keyNumString = keyNum.toString().padEnd(2, '0')
addKey {
- name = "test key $keyNum"
+ name = "test key $keyNumString"
}.build {
addMeta {
self {
(1..20).forEach { tagNum ->
+ val tagNumString = tagNum.toString().padEnd(2, '0')
tags.add(
Tag().apply {
project = projectBuilder.self
- name = "tag $keyNum $tagNum"
+ name = "tag $keyNumString $tagNumString"
}
)
}
diff --git a/backend/data/src/main/kotlin/io/tolgee/service/query_builders/TranslationsViewBuilder.kt b/backend/data/src/main/kotlin/io/tolgee/service/query_builders/TranslationsViewBuilder.kt
index 5ce8b7c292..1b6bd36e6e 100644
--- a/backend/data/src/main/kotlin/io/tolgee/service/query_builders/TranslationsViewBuilder.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/service/query_builders/TranslationsViewBuilder.kt
@@ -1,5 +1,7 @@
package io.tolgee.service.query_builders
+import io.tolgee.configuration.tolgee.DatabaseProperties
+import io.tolgee.configuration.tolgee.TolgeeProperties
import io.tolgee.dtos.request.translation.TranslationFilterByState
import io.tolgee.dtos.request.translation.TranslationFilters
import io.tolgee.dtos.response.CursorValue
@@ -338,9 +340,13 @@ class TranslationsViewBuilder(
): Page<KeyWithTranslationsView> {
val em = applicationContext.getBean(EntityManager::class.java)
val tagService = applicationContext.getBean(TagService::class.java)
+ val properties = applicationContext.getBean(TolgeeProperties::class.java)
+ val isCockroachDb = properties.database.type == DatabaseProperties.DatabaseType.COCKROACH
- // otherwise it takes forever for postgres to plan the execution
- em.createNativeQuery("SET join_collapse_limit TO 1").executeUpdate()
+ if (!isCockroachDb) {
+ // otherwise it takes forever for postgres to plan the execution
+ em.createNativeQuery("SET join_collapse_limit TO 1").executeUpdate()
+ }
var translationsViewBuilder = TranslationsViewBuilder(
cb = em.criteriaBuilder,
@@ -366,8 +372,10 @@ class TranslationsViewBuilder(
}
val views = query.resultList.map { KeyWithTranslationsView.of(it, languages.toList()) }
- // reset the value
- em.createNativeQuery("SET join_collapse_limit TO DEFAULT").executeUpdate()
+ if (!isCockroachDb) {
+ // reset the value
+ em.createNativeQuery("SET join_collapse_limit TO DEFAULT").executeUpdate()
+ }
val keyIds = views.map { it.keyId }
tagService.getTagsForKeyIds(keyIds).let { tagMap ->
diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml
index 702d316db8..b9c25911f9 100644
--- a/e2e/docker-compose.yml
+++ b/e2e/docker-compose.yml
@@ -7,7 +7,7 @@ services:
- 8201:8201
- 8091:8091
environment:
- - spring.profiles.active=docker,e2e
+ - spring.profiles.active=docker,e2e,${SPRING_PROFILES_ACTIVE:-default}
- tolgee.smtp.host=fakesmtp
- tolgee.smtp.port=1025
- tolgee.frontend-url=http://localhost:8201
@@ -18,5 +18,13 @@ services:
ports:
- "21025:1025"
- "21080:1080"
+ cockroachdb:
+ container_name: tolgee_cockroach_e2e
+ image: cockroachdb/cockroach:v22.1.9
+ ports:
+ - 26257:26257
+ command:
+ - start-single-node
+ - --insecure
volumes:
e2e-db-data:
diff --git a/gradle/e2e.gradle b/gradle/e2e.gradle
index fd40094d9f..20bfc8f7a4 100644
--- a/gradle/e2e.gradle
+++ b/gradle/e2e.gradle
@@ -6,6 +6,7 @@ ext {
E2E_DIR = "${project.projectDir}/e2e"
BILLING_E2E_DIR = "${project.projectDir}/../billing/e2e"
WEBAPP_DIR = "${project.projectDir}/webapp"
+ IS_COCKROACH = System.getenv("SPRING_PROFILES_ACTIVE")?.contains("cockroach")
}
@@ -80,7 +81,7 @@ task runE2e(type: Exec, group: "e2e") {
finalizedBy "saveServerLogs", "stopDockerE2e", "cleanupDockerE2e"
}
-task saveServerLogs(type: Exec, group: "e2E"){
+task saveServerLogs(type: Exec, group: "e2E") {
commandLine "bash", "-c", "docker-compose logs > server.log"
workingDir E2E_DIR
}
@@ -122,13 +123,21 @@ task runWebAppNpmStartE2eDev(type: Exec, group: "e2e") {
task runDockerE2e(type: Exec, group: "e2e") {
dependsOn "tagDockerLocal"
- commandLine "docker-compose", "up", "-d"
+ if (IS_COCKROACH) {
+ commandLine "docker-compose", "up", "-d", "fakesmtp", "cockroachdb", "app"
+ } else {
+ commandLine "docker-compose", "up", "-d", "fakesmtp", "app"
+ }
workingDir E2E_DIR
finalizedBy "waitForRunningContainer"
}
task runDockerE2eDev(type: Exec, group: "e2e") {
- commandLine "docker-compose", "up", "-d", "fakesmtp"
+ if (IS_COCKROACH) {
+ commandLine "docker-compose", "up", "-d", "fakesmtp", "cockroachdb"
+ } else {
+ commandLine "docker-compose", "up", "-d", "fakesmtp"
+ }
workingDir E2E_DIR
}
From d5d4d0cc1bb136e76eb717a26858a4ab485e8768 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Sun, 23 Oct 2022 18:51:19 +0200
Subject: [PATCH 06/19] feat: Cockroach DB support > test with cockroach
22.2-beta
---
backend/app/build.gradle | 4 +++-
e2e/docker-compose.yml | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/backend/app/build.gradle b/backend/app/build.gradle
index 7228642489..d85c94ad1e 100644
--- a/backend/app/build.gradle
+++ b/backend/app/build.gradle
@@ -170,9 +170,11 @@ test {
}
task testCockroach(type: Test, group: 'verification') {
+ dependsOn "startCockroachDb"
useJUnitPlatform()
maxHeapSize = "2048m"
systemProperty 'SPRING_PROFILES_ACTIVE', 'cockroach'
+ finalizedBy "stopCockroachDb"
}
task runContextRecreatingTests(type: Test, group: 'verification') {
@@ -193,7 +195,7 @@ task runStandardTests(type: Test, group: 'verification') {
task startCockroachDb(type: Exec){
onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
- commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach:v22.1.9 start-single-node --insecure"
+ commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach-unstable:v22.2.0-beta.4 start-single-node --insecure"
}
task stopCockroachDb(type: Exec){
diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml
index b9c25911f9..5ce09480db 100644
--- a/e2e/docker-compose.yml
+++ b/e2e/docker-compose.yml
@@ -20,7 +20,7 @@ services:
- "21080:1080"
cockroachdb:
container_name: tolgee_cockroach_e2e
- image: cockroachdb/cockroach:v22.1.9
+ image: cockroachdb/cockroach-unstable:v22.2.0-beta.4
ports:
- 26257:26257
command:
From b1f8145ac1b31e8d79e59e692a5b8a7c0bf08068 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Sun, 23 Oct 2022 21:05:51 +0200
Subject: [PATCH 07/19] feat: Cockroach DB support > fix tag tests
---
.github/workflows/test.yml | 2 +-
.../io/tolgee/api/v2/controllers/TagsControllerTest.kt | 6 +++---
.../tolgee/development/testDataBuilder/data/TagsTestData.kt | 4 ++--
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c12a189dde..a89b30828d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -105,7 +105,7 @@ jobs:
- uses: actions/upload-artifact@v2
if: always()
with:
- name: backend_test_reports_${{ steps.version.outputs.reportName }}
+ name: backend_test_reports_${{ matrix.profile }}
path: |
./**/build/reports/**/*
diff --git a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/TagsControllerTest.kt b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/TagsControllerTest.kt
index a4eb4b26cc..fbea144c46 100644
--- a/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/TagsControllerTest.kt
+++ b/backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/TagsControllerTest.kt
@@ -48,8 +48,8 @@ class TagsControllerTest : ProjectAuthControllerTest("/v2/projects/") {
performProjectAuthGet("tags?page=5").andAssertThatJson {
node("_embedded.tags") {
isArray.hasSize(20)
- node("[0].name").isEqualTo("tag 14 12")
- node("[19].name").isEqualTo("tag 15 10")
+ node("[0].name").isEqualTo("tag 05 19")
+ node("[19].name").isEqualTo("tag 06 18")
}
}
}
@@ -61,7 +61,7 @@ class TagsControllerTest : ProjectAuthControllerTest("/v2/projects/") {
node("_embedded.tags") {
isArray.hasSize(20)
node("[0].id").isValidId
- node("[0].name").isEqualTo("tag 11 3")
+ node("[0].name").isEqualTo("tag 02 19")
}
}
}
diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt
index 7768ee523e..6258141509 100644
--- a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/data/TagsTestData.kt
@@ -55,14 +55,14 @@ class TagsTestData : BaseTestData("tagsTestUser", "tagsTestProject") {
}
}
(1..20).forEach { keyNum ->
- val keyNumString = keyNum.toString().padEnd(2, '0')
+ val keyNumString = keyNum.toString().padStart(2, '0')
addKey {
name = "test key $keyNumString"
}.build {
addMeta {
self {
(1..20).forEach { tagNum ->
- val tagNumString = tagNum.toString().padEnd(2, '0')
+ val tagNumString = tagNum.toString().padStart(2, '0')
tags.add(
Tag().apply {
project = projectBuilder.self
From 5405fc35f0d6410b76e2949e546255d2892ce083 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Sun, 23 Oct 2022 21:09:36 +0200
Subject: [PATCH 08/19] feat: Cockroach DB support > run cockroach in memory
for testing
---
backend/app/build.gradle | 2 +-
e2e/docker-compose.yml | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/backend/app/build.gradle b/backend/app/build.gradle
index d85c94ad1e..38d0afc7e0 100644
--- a/backend/app/build.gradle
+++ b/backend/app/build.gradle
@@ -195,7 +195,7 @@ task runStandardTests(type: Test, group: 'verification') {
task startCockroachDb(type: Exec){
onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
- commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach-unstable:v22.2.0-beta.4 start-single-node --insecure"
+ commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach-unstable:v22.2.0-beta.4 start-single-node --insecure --store=type=mem,size=0.25"
}
task stopCockroachDb(type: Exec){
diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml
index 5ce09480db..9b52318d6d 100644
--- a/e2e/docker-compose.yml
+++ b/e2e/docker-compose.yml
@@ -26,5 +26,6 @@ services:
command:
- start-single-node
- --insecure
+ - --store=type=mem,size=0.25
volumes:
e2e-db-data:
From ab73ffe63bc235be7948433921ef4828e4dfa2f2 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Thu, 10 Nov 2022 14:14:57 +0100
Subject: [PATCH 09/19] fix: Update CockroachDB versions
---
backend/app/build.gradle | 2 +-
e2e/docker-compose.yml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/backend/app/build.gradle b/backend/app/build.gradle
index 38d0afc7e0..de893e1fdc 100644
--- a/backend/app/build.gradle
+++ b/backend/app/build.gradle
@@ -195,7 +195,7 @@ task runStandardTests(type: Test, group: 'verification') {
task startCockroachDb(type: Exec){
onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
- commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach-unstable:v22.2.0-beta.4 start-single-node --insecure --store=type=mem,size=0.25"
+ commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach-unstable:v22.2.0-rc.1 start-single-node --insecure --store=type=mem,size=0.25"
}
task stopCockroachDb(type: Exec){
diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml
index 9b52318d6d..90055b613d 100644
--- a/e2e/docker-compose.yml
+++ b/e2e/docker-compose.yml
@@ -20,7 +20,7 @@ services:
- "21080:1080"
cockroachdb:
container_name: tolgee_cockroach_e2e
- image: cockroachdb/cockroach-unstable:v22.2.0-beta.4
+ image: cockroachdb/cockroach-unstable:v22.2.0-rc.1
ports:
- 26257:26257
command:
From 171b4c5c98740094984e1b12277e66cbc910236c Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Wed, 14 Dec 2022 14:35:14 +0100
Subject: [PATCH 10/19] fix: Update cockroach db to 22.0 stable version
---
backend/app/build.gradle | 2 +-
e2e/docker-compose.yml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/backend/app/build.gradle b/backend/app/build.gradle
index de893e1fdc..ebdf44e5cf 100644
--- a/backend/app/build.gradle
+++ b/backend/app/build.gradle
@@ -195,7 +195,7 @@ task runStandardTests(type: Test, group: 'verification') {
task startCockroachDb(type: Exec){
onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
- commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach-unstable:v22.2.0-rc.1 start-single-node --insecure --store=type=mem,size=0.25"
+ commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach-unstable:v22.2.0 start-single-node --insecure --store=type=mem,size=0.25"
}
task stopCockroachDb(type: Exec){
diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml
index 90055b613d..adf0083630 100644
--- a/e2e/docker-compose.yml
+++ b/e2e/docker-compose.yml
@@ -20,7 +20,7 @@ services:
- "21080:1080"
cockroachdb:
container_name: tolgee_cockroach_e2e
- image: cockroachdb/cockroach-unstable:v22.2.0-rc.1
+ image: cockroachdb/cockroach-unstable:v22.2.0
ports:
- 26257:26257
command:
From 9d9a4834ee42080429b07b57c0124d18860d8d2f Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Wed, 14 Dec 2022 18:16:02 +0100
Subject: [PATCH 11/19] fix: Update cockroach db to 22.0 stable version
---
backend/app/build.gradle | 2 +-
e2e/docker-compose.yml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/backend/app/build.gradle b/backend/app/build.gradle
index ebdf44e5cf..64733befdf 100644
--- a/backend/app/build.gradle
+++ b/backend/app/build.gradle
@@ -195,7 +195,7 @@ task runStandardTests(type: Test, group: 'verification') {
task startCockroachDb(type: Exec){
onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
- commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach-unstable:v22.2.0 start-single-node --insecure --store=type=mem,size=0.25"
+ commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach:v22.2.0 start-single-node --insecure --store=type=mem,size=0.25"
}
task stopCockroachDb(type: Exec){
diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml
index adf0083630..f1eac1c677 100644
--- a/e2e/docker-compose.yml
+++ b/e2e/docker-compose.yml
@@ -20,7 +20,7 @@ services:
- "21080:1080"
cockroachdb:
container_name: tolgee_cockroach_e2e
- image: cockroachdb/cockroach-unstable:v22.2.0
+ image: cockroachdb/cockroach:v22.2.0
ports:
- 26257:26257
command:
From 644d584cd4ed098d64b64c7797b00e845a6e5e99 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Tue, 3 Jan 2023 11:23:05 +0100
Subject: [PATCH 12/19] fix: Billing and Cockroach
---
backend/app/build.gradle | 18 ------------------
build.gradle | 10 ++++++++++
2 files changed, 10 insertions(+), 18 deletions(-)
diff --git a/backend/app/build.gradle b/backend/app/build.gradle
index 64733befdf..441e4c0922 100644
--- a/backend/app/build.gradle
+++ b/backend/app/build.gradle
@@ -169,14 +169,6 @@ test {
maxHeapSize = "2048m"
}
-task testCockroach(type: Test, group: 'verification') {
- dependsOn "startCockroachDb"
- useJUnitPlatform()
- maxHeapSize = "2048m"
- systemProperty 'SPRING_PROFILES_ACTIVE', 'cockroach'
- finalizedBy "stopCockroachDb"
-}
-
task runContextRecreatingTests(type: Test, group: 'verification') {
dependsOn "startCockroachDb"
useJUnitPlatform {
@@ -193,16 +185,6 @@ task runStandardTests(type: Test, group: 'verification') {
finalizedBy "stopCockroachDb"
}
-task startCockroachDb(type: Exec){
- onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
- commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach:v22.2.0 start-single-node --insecure --store=type=mem,size=0.25"
-}
-
-task stopCockroachDb(type: Exec){
- onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
- commandLine "bash", "-c", "docker stop roach"
-}
-
springBoot {
buildInfo {
properties {
diff --git a/build.gradle b/build.gradle
index a27640244c..c61102ce32 100644
--- a/build.gradle
+++ b/build.gradle
@@ -147,4 +147,14 @@ task ktlintFormat(type: JavaExec, group: "formatting") {
subprojects {
task allDeps(type: DependencyReportTask) {}
+
+ task startCockroachDb(type: Exec){
+ onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
+ commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach:v22.2.1 start-single-node --insecure --store=type=mem,size=0.25"
+ }
+
+ task stopCockroachDb(type: Exec){
+ onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
+ commandLine "bash", "-c", "docker stop roach"
+ }
}
From 04cdb60687a9d5dbe0959d7db27636db152616f4 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Tue, 3 Jan 2023 15:54:59 +0100
Subject: [PATCH 13/19] fix: Billing and Cockroach > Fix deadlock
---
.../src/main/kotlin/io/tolgee/Application.kt | 2 ++
.../api/v2/controllers/V2ImportController.kt | 2 +-
.../resources/application-devcockroach.yaml | 3 ++
.../testDataBuilder/TestDataService.kt | 2 ++
.../io/tolgee/service/LanguageService.kt | 3 ++
.../kotlin/io/tolgee/util/transactionUtil.kt | 1 +
.../io/tolgee/util/withTimeoutAndRetry.kt | 28 +++++++++++++++++++
build.gradle | 2 +-
8 files changed, 41 insertions(+), 2 deletions(-)
create mode 100644 backend/app/src/main/resources/application-devcockroach.yaml
create mode 100644 backend/data/src/main/kotlin/io/tolgee/util/withTimeoutAndRetry.kt
diff --git a/backend/app/src/main/kotlin/io/tolgee/Application.kt b/backend/app/src/main/kotlin/io/tolgee/Application.kt
index 53f90bbeb3..a8e8cd3013 100644
--- a/backend/app/src/main/kotlin/io/tolgee/Application.kt
+++ b/backend/app/src/main/kotlin/io/tolgee/Application.kt
@@ -20,6 +20,7 @@ import org.springframework.boot.autoconfigure.ldap.LdapAutoConfiguration
import org.springframework.boot.context.properties.ConfigurationPropertiesScan
import org.springframework.data.jpa.repository.config.EnableJpaAuditing
import org.springframework.data.jpa.repository.config.EnableJpaRepositories
+import org.springframework.retry.annotation.EnableRetry
@SpringBootApplication(
exclude = [
@@ -43,6 +44,7 @@ import org.springframework.data.jpa.repository.config.EnableJpaRepositories
@EntityScan("io.tolgee.model")
@ConfigurationPropertiesScan
@EnableJpaRepositories("io.tolgee.repository")
+@EnableRetry
class Application {
companion object {
@JvmStatic
diff --git a/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt b/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt
index e0b8165893..50eddaeade 100644
--- a/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt
+++ b/backend/app/src/main/kotlin/io/tolgee/api/v2/controllers/V2ImportController.kt
@@ -350,7 +350,7 @@ class V2ImportController(
}
private fun checkLanguageFromProject(languageId: Long): Language {
- val existingLanguage = languageService.findById(languageId).orElse(null) ?: throw NotFoundException()
+ val existingLanguage = languageService.find(languageId) ?: throw NotFoundException()
if (existingLanguage.project.id != projectHolder.project.id) {
throw BadRequestException(io.tolgee.constants.Message.IMPORT_LANGUAGE_NOT_FROM_PROJECT)
}
diff --git a/backend/app/src/main/resources/application-devcockroach.yaml b/backend/app/src/main/resources/application-devcockroach.yaml
new file mode 100644
index 0000000000..0ca0de1d63
--- /dev/null
+++ b/backend/app/src/main/resources/application-devcockroach.yaml
@@ -0,0 +1,3 @@
+spring:
+ datasource:
+ url: jdbc:postgresql://localhost:26257/postgres
diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt
index 98e844cf66..a530b752c1 100644
--- a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt
@@ -256,6 +256,8 @@ class TestDataService(
projectBuilders.forEach { projectBuilder ->
executeInNewTransaction(transactionManager) {
projectService.save(projectBuilder.self)
+ }
+ executeInNewTransaction(transactionManager) {
saveAllProjectDependants(projectBuilder)
}
}
diff --git a/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt b/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt
index cabe60ca0b..ecdaf174d2 100644
--- a/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/service/LanguageService.kt
@@ -16,6 +16,7 @@ import org.springframework.data.domain.Page
import org.springframework.data.domain.PageRequest
import org.springframework.data.domain.Pageable
import org.springframework.data.domain.Sort
+import org.springframework.retry.annotation.Retryable
import org.springframework.stereotype.Service
import org.springframework.transaction.annotation.Transactional
import java.util.*
@@ -78,6 +79,8 @@ class LanguageService(
return find(id) ?: throw NotFoundException(Message.LANGUAGE_NOT_FOUND)
}
+ @Transactional
+ @Retryable()
fun find(id: Long): Language? {
return languageRepository.findById(id).orElse(null)
}
diff --git a/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt b/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt
index a581438d07..007c8aea94 100644
--- a/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt
@@ -7,6 +7,7 @@ import org.springframework.transaction.support.TransactionTemplate
fun <T> executeInNewTransaction(transactionManager: PlatformTransactionManager, fn: () -> T): T {
val tt = TransactionTemplate(transactionManager)
tt.propagationBehavior = TransactionDefinition.PROPAGATION_REQUIRES_NEW
+ tt.isolationLevel = TransactionDefinition.ISOLATION_SERIALIZABLE
return tt.execute {
fn()
diff --git a/backend/data/src/main/kotlin/io/tolgee/util/withTimeoutAndRetry.kt b/backend/data/src/main/kotlin/io/tolgee/util/withTimeoutAndRetry.kt
new file mode 100644
index 0000000000..6219636842
--- /dev/null
+++ b/backend/data/src/main/kotlin/io/tolgee/util/withTimeoutAndRetry.kt
@@ -0,0 +1,28 @@
+package io.tolgee.util
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.TimeoutCancellationException
+import kotlinx.coroutines.async
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withTimeout
+
+fun <T> withTimeoutAndRetry(timeoutInMs: Long = 4000, fn: () -> T): T {
+ return runBlocking(Dispatchers.IO) {
+ val retries = 3
+ var exp: TimeoutCancellationException? = null
+ (0..retries).forEach {
+ val task = async { fn() }
+ try {
+ return@runBlocking withTimeout(timeoutInMs) {
+ task.await()
+ }
+ } catch (e: TimeoutCancellationException) {
+ exp = e
+ }
+ }
+ exp?.let {
+ throw it
+ }
+ throw IllegalStateException("No exception caught!")
+ }
+}
diff --git a/build.gradle b/build.gradle
index c61102ce32..a6c776cc54 100644
--- a/build.gradle
+++ b/build.gradle
@@ -150,7 +150,7 @@ subprojects {
task startCockroachDb(type: Exec){
onlyIf { System.getenv("SPRING_PROFILES_ACTIVE") == "cockroach" }
- commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 cockroachdb/cockroach:v22.2.1 start-single-node --insecure --store=type=mem,size=0.25"
+ commandLine "bash", "-c", "docker run --rm -d --name=roach -p 26257:26257 -p 8089:8080 cockroachdb/cockroach:v22.2.1 start-single-node --insecure --store=type=mem,size=0.25"
}
task stopCockroachDb(type: Exec){
From 429e487028e1bfb7e83dd8c9728862cd42b3d8fa Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Wed, 4 Jan 2023 10:48:42 +0100
Subject: [PATCH 14/19] fix: Billing and Cockroach > transactionUtil
---
backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt | 1 -
1 file changed, 1 deletion(-)
diff --git a/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt b/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt
index 007c8aea94..a581438d07 100644
--- a/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/util/transactionUtil.kt
@@ -7,7 +7,6 @@ import org.springframework.transaction.support.TransactionTemplate
fun <T> executeInNewTransaction(transactionManager: PlatformTransactionManager, fn: () -> T): T {
val tt = TransactionTemplate(transactionManager)
tt.propagationBehavior = TransactionDefinition.PROPAGATION_REQUIRES_NEW
- tt.isolationLevel = TransactionDefinition.ISOLATION_SERIALIZABLE
return tt.execute {
fn()
From 866f2af5619d6ccbd86d0a4317db9b74b4886912 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Wed, 4 Jan 2023 14:22:30 +0100
Subject: [PATCH 15/19] fix: Cockroach > Fix Auto translation test
---
.../io/tolgee/development/testDataBuilder/TestDataService.kt | 2 --
1 file changed, 2 deletions(-)
diff --git a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt
index a530b752c1..98e844cf66 100644
--- a/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/development/testDataBuilder/TestDataService.kt
@@ -256,8 +256,6 @@ class TestDataService(
projectBuilders.forEach { projectBuilder ->
executeInNewTransaction(transactionManager) {
projectService.save(projectBuilder.self)
- }
- executeInNewTransaction(transactionManager) {
saveAllProjectDependants(projectBuilder)
}
}
From e669e058556ba0e2a4c3a843ef3461c7a40976cc Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Wed, 4 Jan 2023 18:01:27 +0100
Subject: [PATCH 16/19] fix: Cockroach > Add entity listening using
interceptors, no new transaction in listener
---
.../iterceptor/ActivityInterceptor.kt | 6 +++++
.../iterceptor/PreCommitEventPublisher.kt | 22 ++++++++++++++++
.../io/tolgee/events/OnEntityPreDelete.kt | 9 +++++++
.../io/tolgee/events/OnEntityPrePersist.kt | 9 +++++++
.../io/tolgee/events/OnEntityPreUpdate.kt | 9 +++++++
.../service/OrganizationStatsService.kt | 26 ++++++++++++++-----
.../io/tolgee/service/ScreenshotService.kt | 26 ++++++++++++++-----
.../tolgee/service/project/ProjectService.kt | 1 +
e2e/docker-compose.yml | 1 +
9 files changed, 95 insertions(+), 14 deletions(-)
create mode 100644 backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt
create mode 100644 backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreDelete.kt
create mode 100644 backend/data/src/main/kotlin/io/tolgee/events/OnEntityPrePersist.kt
create mode 100644 backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreUpdate.kt
diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt
index 5cdf98d831..cc412b6813 100644
--- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/ActivityInterceptor.kt
@@ -26,6 +26,7 @@ class ActivityInterceptor : EmptyInterceptor() {
propertyNames: Array<out String>?,
types: Array<out Type>?
): Boolean {
+ preCommitEventsPublisher.onPersist(entity)
interceptedEventsManager.onFieldModificationsActivity(
entity, state, null, propertyNames, RevisionType.ADD
)
@@ -39,6 +40,7 @@ class ActivityInterceptor : EmptyInterceptor() {
propertyNames: Array<out String>?,
types: Array<out Type>?
) {
+ preCommitEventsPublisher.onDelete(entity)
interceptedEventsManager.onFieldModificationsActivity(
entity, null, state, propertyNames, RevisionType.DEL
)
@@ -52,6 +54,7 @@ class ActivityInterceptor : EmptyInterceptor() {
propertyNames: Array<out String>?,
types: Array<out Type>?
): Boolean {
+ preCommitEventsPublisher.onUpdate(entity)
interceptedEventsManager.onFieldModificationsActivity(
entity,
currentState,
@@ -76,4 +79,7 @@ class ActivityInterceptor : EmptyInterceptor() {
val interceptedEventsManager: InterceptedEventsManager
get() = applicationContext.getBean(InterceptedEventsManager::class.java)
+
+ val preCommitEventsPublisher: PreCommitEventPublisher
+ get() = applicationContext.getBean(PreCommitEventPublisher::class.java)
}
diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt
new file mode 100644
index 0000000000..80319fd533
--- /dev/null
+++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt
@@ -0,0 +1,22 @@
+package io.tolgee.activity.iterceptor
+
+import io.tolgee.events.OnEntityPreDelete
+import io.tolgee.events.OnEntityPreUpdate
+import org.springframework.context.ApplicationContext
+import org.springframework.stereotype.Component
+
+@Component
+class PreCommitEventPublisher(private val applicationContext: ApplicationContext) {
+
+ fun onPersist(entity: Any?) {
+ applicationContext.publishEvent(OnEntityPreDelete(this, entity))
+ }
+
+ fun onUpdate(entity: Any?) {
+ applicationContext.publishEvent(OnEntityPreUpdate(this, entity))
+ }
+
+ fun onDelete(entity: Any?) {
+ applicationContext.publishEvent(OnEntityPreDelete(this, entity))
+ }
+}
diff --git a/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreDelete.kt b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreDelete.kt
new file mode 100644
index 0000000000..3a6186476e
--- /dev/null
+++ b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreDelete.kt
@@ -0,0 +1,9 @@
+package io.tolgee.events
+
+import io.tolgee.activity.iterceptor.PreCommitEventPublisher
+import org.springframework.context.ApplicationEvent
+
+class OnEntityPreDelete(
+ val source: PreCommitEventPublisher,
+ val entity: Any?
+) : ApplicationEvent(source)
diff --git a/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPrePersist.kt b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPrePersist.kt
new file mode 100644
index 0000000000..bece6b98a1
--- /dev/null
+++ b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPrePersist.kt
@@ -0,0 +1,9 @@
+package io.tolgee.events
+
+import io.tolgee.activity.iterceptor.PreCommitEventPublisher
+import org.springframework.context.ApplicationEvent
+
+class OnEntityPrePersist(
+ val source: PreCommitEventPublisher,
+ val entity: Any?
+) : ApplicationEvent(source)
diff --git a/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreUpdate.kt b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreUpdate.kt
new file mode 100644
index 0000000000..3632df6bf7
--- /dev/null
+++ b/backend/data/src/main/kotlin/io/tolgee/events/OnEntityPreUpdate.kt
@@ -0,0 +1,9 @@
+package io.tolgee.events
+
+import io.tolgee.activity.iterceptor.PreCommitEventPublisher
+import org.springframework.context.ApplicationEvent
+
+class OnEntityPreUpdate(
+ val source: PreCommitEventPublisher,
+ val entity: Any?
+) : ApplicationEvent(source)
diff --git a/backend/data/src/main/kotlin/io/tolgee/service/OrganizationStatsService.kt b/backend/data/src/main/kotlin/io/tolgee/service/OrganizationStatsService.kt
index fba08a8857..47456dcc7d 100644
--- a/backend/data/src/main/kotlin/io/tolgee/service/OrganizationStatsService.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/service/OrganizationStatsService.kt
@@ -1,12 +1,15 @@
package io.tolgee.service
+import org.hibernate.FlushMode
+import org.hibernate.Session
import org.springframework.stereotype.Service
import java.math.BigDecimal
import javax.persistence.EntityManager
+import javax.persistence.FlushModeType
@Service
class OrganizationStatsService(
- private val entityManager: EntityManager
+ private val entityManager: EntityManager,
) {
fun getProjectLanguageCount(projectId: Long): Long {
return entityManager
@@ -33,9 +36,13 @@ class OrganizationStatsService(
}
fun getCurrentTranslationCount(organizationId: Long): Long {
- val result = entityManager.createNativeQuery(
- """
- select
+ val session = entityManager.unwrap(Session::class.java)
+ return try {
+ session.hibernateFlushMode = FlushMode.MANUAL
+ session.flushMode = FlushModeType.COMMIT
+ val query = session.createNativeQuery(
+ """
+ select
(select sum(keyCount * languageCount) as translationCount
from (select p.id as projectId, count(l.id) as languageCount
from project as p
@@ -47,8 +54,13 @@ class OrganizationStatsService(
join key as k on k.project_id = p.id
where p.organization_owner_id = :organizationId
group by p.id) as keyCounts on keyCounts.projectId = languageCounts.projectId)
- """.trimIndent()
- ).setParameter("organizationId", organizationId).singleResult as BigDecimal? ?: 0
- return result.toLong()
+ """.trimIndent()
+ ).setParameter("organizationId", organizationId)
+ val result = query.singleResult as BigDecimal? ?: 0
+ result.toLong()
+ } finally {
+ session.hibernateFlushMode = FlushMode.AUTO
+ session.flushMode = FlushModeType.AUTO
+ }
}
}
diff --git a/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt b/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt
index c362074e59..a0b039aab7 100644
--- a/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt
@@ -15,8 +15,11 @@ import io.tolgee.repository.ScreenshotRepository
import io.tolgee.security.AuthenticationFacade
import io.tolgee.service.ImageUploadService.Companion.UPLOADED_IMAGES_STORAGE_FOLDER_NAME
import io.tolgee.util.ImageConverter
+import io.tolgee.util.executeInNewTransaction
import org.springframework.core.io.InputStreamSource
+import org.springframework.retry.annotation.Retryable
import org.springframework.stereotype.Service
+import org.springframework.transaction.PlatformTransactionManager
import org.springframework.transaction.annotation.Transactional
@Service
@@ -25,13 +28,13 @@ class ScreenshotService(
private val fileStorage: FileStorage,
private val tolgeeProperties: TolgeeProperties,
private val imageUploadService: ImageUploadService,
- private val authenticationFacade: AuthenticationFacade
+ private val authenticationFacade: AuthenticationFacade,
+ private val transactionManager: PlatformTransactionManager
) {
companion object {
const val SCREENSHOTS_STORAGE_FOLDER_NAME = "screenshots"
}
- @Transactional
fun store(screenshotImage: InputStreamSource, key: Key): Screenshot {
if (getScreenshotsCountForKey(key) >= tolgeeProperties.maxScreenshotsPerKey) {
throw BadRequestException(
@@ -45,16 +48,25 @@ class ScreenshotService(
return storeProcessed(image.toByteArray(), thumbnail.toByteArray(), key)
}
+ @Retryable
fun storeProcessed(image: ByteArray, thumbnail: ByteArray, key: Key): Screenshot {
val screenshotEntity = Screenshot().also {
it.key = key
it.extension = "png"
}
- key.screenshots.add(screenshotEntity)
- screenshotRepository.save(screenshotEntity)
- fileStorage.storeFile(screenshotEntity.getThumbnailPath(), thumbnail)
- fileStorage.storeFile(screenshotEntity.getFilePath(), image)
- return screenshotEntity
+ try {
+ return executeInNewTransaction(transactionManager) {
+ key.screenshots.add(screenshotEntity)
+ screenshotRepository.save(screenshotEntity)
+ fileStorage.storeFile(screenshotEntity.getThumbnailPath(), thumbnail)
+ fileStorage.storeFile(screenshotEntity.getFilePath(), image)
+ screenshotEntity
+ }
+ } catch (e: Exception) {
+ fileStorage.deleteFile(screenshotEntity.getThumbnailPath())
+ fileStorage.deleteFile(screenshotEntity.getFilePath())
+ throw e
+ }
}
@Transactional
diff --git a/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt b/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt
index 62ec8779ae..69fb6a1885 100644
--- a/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/service/project/ProjectService.kt
@@ -311,6 +311,7 @@ class ProjectService constructor(
projectRepository.saveAll(projects)
@CacheEvict(cacheNames = [Caches.PROJECTS], key = "#result.id")
+ @Transactional
fun save(project: Project): Project {
val isCreating = project.id == 0L
projectRepository.save(project)
diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml
index f1eac1c677..8e7466cd03 100644
--- a/e2e/docker-compose.yml
+++ b/e2e/docker-compose.yml
@@ -23,6 +23,7 @@ services:
image: cockroachdb/cockroach:v22.2.0
ports:
- 26257:26257
+ - 8088:8080
command:
- start-single-node
- --insecure
From a2ba36d2e1df5e8d7b4617248070d18ed783855e Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Thu, 5 Jan 2023 09:18:54 +0100
Subject: [PATCH 17/19] fix: Cockroach > TestListener
---
.../service/CockroachStatsTestListener.kt | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
create mode 100644 backend/data/src/main/kotlin/io/tolgee/service/CockroachStatsTestListener.kt
diff --git a/backend/data/src/main/kotlin/io/tolgee/service/CockroachStatsTestListener.kt b/backend/data/src/main/kotlin/io/tolgee/service/CockroachStatsTestListener.kt
new file mode 100644
index 0000000000..3175a9054d
--- /dev/null
+++ b/backend/data/src/main/kotlin/io/tolgee/service/CockroachStatsTestListener.kt
@@ -0,0 +1,19 @@
+package io.tolgee.service
+
+import io.tolgee.events.OnEntityPrePersist
+import io.tolgee.model.Language
+import org.springframework.context.event.EventListener
+import org.springframework.stereotype.Service
+
+@Service
+class CockroachStatsTestListener(
+ private val organizationStatsService: OrganizationStatsService
+) {
+
+ @EventListener
+ fun onLanguageAdd(event: OnEntityPrePersist) {
+ (event.entity as? Language)?.let {
+ organizationStatsService.getCurrentTranslationCount(event.entity.project.organizationOwner.id)
+ }
+ }
+}
From 03fd50cddd31d03256c8c7f74989cdf9ed5fc536 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Thu, 5 Jan 2023 09:18:54 +0100
Subject: [PATCH 18/19] fix: Cockroach > TestListener
---
.../io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt
index 80319fd533..5d36d5e5d5 100644
--- a/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/activity/iterceptor/PreCommitEventPublisher.kt
@@ -1,6 +1,7 @@
package io.tolgee.activity.iterceptor
import io.tolgee.events.OnEntityPreDelete
+import io.tolgee.events.OnEntityPrePersist
import io.tolgee.events.OnEntityPreUpdate
import org.springframework.context.ApplicationContext
import org.springframework.stereotype.Component
@@ -9,7 +10,7 @@ import org.springframework.stereotype.Component
class PreCommitEventPublisher(private val applicationContext: ApplicationContext) {
fun onPersist(entity: Any?) {
- applicationContext.publishEvent(OnEntityPreDelete(this, entity))
+ applicationContext.publishEvent(OnEntityPrePersist(this, entity))
}
fun onUpdate(entity: Any?) {
From adbb2cf1884b4cc1b7b05410a1a2574a890f1f82 Mon Sep 17 00:00:00 2001
From: Jan Cizmar <cizmar@chlupac.com>
Date: Thu, 5 Jan 2023 11:02:49 +0100
Subject: [PATCH 19/19] fix: Cockroach > Fix tests
---
.../service/CockroachStatsTestListener.kt | 19 ----------
.../io/tolgee/service/ScreenshotService.kt | 38 +++++++++++--------
2 files changed, 23 insertions(+), 34 deletions(-)
delete mode 100644 backend/data/src/main/kotlin/io/tolgee/service/CockroachStatsTestListener.kt
diff --git a/backend/data/src/main/kotlin/io/tolgee/service/CockroachStatsTestListener.kt b/backend/data/src/main/kotlin/io/tolgee/service/CockroachStatsTestListener.kt
deleted file mode 100644
index 3175a9054d..0000000000
--- a/backend/data/src/main/kotlin/io/tolgee/service/CockroachStatsTestListener.kt
+++ /dev/null
@@ -1,19 +0,0 @@
-package io.tolgee.service
-
-import io.tolgee.events.OnEntityPrePersist
-import io.tolgee.model.Language
-import org.springframework.context.event.EventListener
-import org.springframework.stereotype.Service
-
-@Service
-class CockroachStatsTestListener(
- private val organizationStatsService: OrganizationStatsService
-) {
-
- @EventListener
- fun onLanguageAdd(event: OnEntityPrePersist) {
- (event.entity as? Language)?.let {
- organizationStatsService.getCurrentTranslationCount(event.entity.project.organizationOwner.id)
- }
- }
-}
diff --git a/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt b/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt
index a0b039aab7..c11ff65f61 100644
--- a/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt
+++ b/backend/data/src/main/kotlin/io/tolgee/service/ScreenshotService.kt
@@ -35,6 +35,11 @@ class ScreenshotService(
const val SCREENSHOTS_STORAGE_FOLDER_NAME = "screenshots"
}
+ /**
+ * CockroachDB has issues with uploading multiple screenshots in the same time, so we
+ * need to make it retryable and executed in new transaction
+ */
+ @Retryable
fun store(screenshotImage: InputStreamSource, key: Key): Screenshot {
if (getScreenshotsCountForKey(key) >= tolgeeProperties.maxScreenshotsPerKey) {
throw BadRequestException(
@@ -45,28 +50,31 @@ class ScreenshotService(
val converter = ImageConverter(screenshotImage.inputStream)
val image = converter.getImage()
val thumbnail = converter.getThumbNail()
- return storeProcessed(image.toByteArray(), thumbnail.toByteArray(), key)
+ var screenshotEntity: Screenshot? = null
+ try {
+ return executeInNewTransaction(transactionManager) {
+ screenshotEntity = storeProcessed(image.toByteArray(), thumbnail.toByteArray(), key)
+ screenshotEntity!!
+ }
+ } catch (e: Exception) {
+ screenshotEntity?.id?.let {
+ fileStorage.deleteFile(screenshotEntity!!.getThumbnailPath())
+ fileStorage.deleteFile(screenshotEntity!!.getFilePath())
+ }
+ throw e
+ }
}
- @Retryable
fun storeProcessed(image: ByteArray, thumbnail: ByteArray, key: Key): Screenshot {
val screenshotEntity = Screenshot().also {
it.key = key
it.extension = "png"
}
- try {
- return executeInNewTransaction(transactionManager) {
- key.screenshots.add(screenshotEntity)
- screenshotRepository.save(screenshotEntity)
- fileStorage.storeFile(screenshotEntity.getThumbnailPath(), thumbnail)
- fileStorage.storeFile(screenshotEntity.getFilePath(), image)
- screenshotEntity
- }
- } catch (e: Exception) {
- fileStorage.deleteFile(screenshotEntity.getThumbnailPath())
- fileStorage.deleteFile(screenshotEntity.getFilePath())
- throw e
- }
+ key.screenshots.add(screenshotEntity)
+ screenshotRepository.save(screenshotEntity)
+ fileStorage.storeFile(screenshotEntity.getThumbnailPath(), thumbnail)
+ fileStorage.storeFile(screenshotEntity.getFilePath(), image)
+ return screenshotEntity
}
@Transactional