Skip to content

Commit b3dd497

Browse files
anatolystansleranatolystansler
authored
Merge pull request #202 from sourcerer-io/develop
feat: release new processing flow
2 parents 4eae376 + 1af51c3 commit b3dd497

File tree

14 files changed

+292
-32
lines changed

14 files changed

+292
-32
lines changed

‎build.gradle

+4-3
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ buildConfig {
3636
buildConfigField 'String', 'PROFILE_URL', 'https://sourcerer.io/'
3737

3838
// App version.
39-
buildConfigField 'int', 'VERSION_CODE', '6'
40-
buildConfigField 'String', 'VERSION', '0.2.1'
39+
buildConfigField 'int', 'VERSION_CODE', '7'
40+
buildConfigField 'String', 'VERSION', '0.2.2'
4141

4242
// Logging.
4343
buildConfigField 'String', 'ENV', project.hasProperty('env') ? env : 'production'
@@ -48,7 +48,7 @@ buildConfig {
4848
buildConfigField 'boolean', 'PRINT_STACK_TRACE', 'false'
4949

5050
// Google Analytics.
51-
buildConfigField 'String', 'GA_BASE_PATH', 'http://www.google-analytics.com'
51+
buildConfigField 'String', 'GA_BASE_PATH', 'https://www.google-analytics.com'
5252
buildConfigField 'String', 'GA_TRACKING_ID', 'UA-107129190-2'
5353
buildConfigField 'boolean', 'IS_GA_ENABLED', 'true'
5454

@@ -59,6 +59,7 @@ buildConfig {
5959
buildConfigField 'boolean', 'COMMIT_HASHER_ENABLED', project.hasProperty('commit-hasher-enabled') ? project.property('commit-hasher-enabled').toString() : 'true'
6060
buildConfigField 'boolean', 'FACT_HASHER_ENABLED', project.hasProperty('fact-hasher-enabled') ? project.property('fact-hasher-enabled').toString() : 'true'
6161
buildConfigField 'boolean', 'LONGEVITY_ENABLED', project.hasProperty('longevity-enabled') ? project.property('longevity-enabled').toString() : 'false'
62+
buildConfigField 'long', 'HEARTBEAT_RATE', project.hasProperty('heartbeat-rate') ? project.property('heartbeat-rate').toString() : '60000'
6263

6364
buildConfig
6465
}

‎src/main/kotlin/app/Analytics.kt

+5-2
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,11 @@ object Analytics {
7272

7373
try {
7474
// Send event to GA with united params.
75-
post(params + defaultParams.filter { !params.contains(it) }
76-
+ idParams).responseString()
75+
val (_, _, result) = post(params +
76+
defaultParams.filter { !params.contains(it) } +
77+
idParams).responseString()
78+
val (_, e) = result
79+
if (e != null) { throw e }
7780
} catch (e: Throwable) {
7881
Logger.error(e, "Error while sending GA report", logOnly = true)
7982
}

‎src/main/kotlin/app/api/Api.kt

+11-1
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,31 @@ package app.api
66
import app.model.Author
77
import app.model.Commit
88
import app.model.Fact
9+
import app.model.Process
10+
import app.model.ProcessEntry
911
import app.model.Repo
1012
import app.model.User
1113

1214
interface Api {
1315
companion object {
1416
val OUT_OF_DATE = 1
17+
val PROCESS_STATUS_START = 100
18+
val PROCESS_STATUS_COMPLETE = 200
19+
val PROCESS_STATUS_FAIL = 1000
20+
val PROCESS_ERROR_RESTRICTED = 2
21+
val PROCESS_ERROR_TOO_BIG = 3
22+
val PROCESS_ERROR_TOO_MUCH_COMMITS = 4
23+
val PROCESS_ERROR_NO_COMMITS = 5
1524
}
1625

1726
fun authorize(): Result<Unit>
1827
fun getUser(): Result<User>
1928
fun postUser(user: User): Result<Unit>
2029
fun postRepo(repo: Repo): Result<Repo>
21-
fun postComplete(): Result<Unit>
2230
fun postCommits(commitsList: List<Commit>): Result<Unit>
2331
fun deleteCommits(commitsList: List<Commit>): Result<Unit>
2432
fun postFacts(factsList: List<Fact>): Result<Unit>
2533
fun postAuthors(authorsList: List<Author>): Result<Unit>
34+
fun postProcessCreate(requestNumEntries: Int): Result<Process>
35+
fun postProcess(processEntries: List<ProcessEntry>): Result<Unit>
2636
}

‎src/main/kotlin/app/api/MockApi.kt

+21-8
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,23 @@ import app.model.Author
88
import app.model.Commit
99
import app.model.Repo
1010
import app.model.Fact
11+
import app.model.Process
12+
import app.model.ProcessEntry
1113
import app.model.User
1214

1315
class MockApi( // GET requests.
1416
var mockUser: User = User(),
15-
var mockRepo: Repo = Repo()) : Api {
17+
var mockRepo: Repo = Repo(),
18+
var mockProcessEntries: List<ProcessEntry> = listOf()) : Api {
1619
// POST requests.
1720
// In case of multiple requests.
1821
var receivedRepos: MutableList<Repo> = mutableListOf()
1922
var receivedAddedCommits: MutableList<Commit> = mutableListOf()
2023
var receivedFacts: MutableList<Fact> = mutableListOf()
2124
var receivedAuthors: MutableList<Author> = mutableListOf()
2225
var receivedUsers: MutableList<User> = mutableListOf()
23-
var receivedComplete: Int = 0
26+
var receivedProcessCreate: MutableList<Process> = mutableListOf()
27+
var receivedProcess: MutableList<Process> = mutableListOf()
2428

2529
// DELETE requests.
2630
var receivedDeletedCommits: MutableList<Commit> = mutableListOf()
@@ -47,12 +51,6 @@ class MockApi( // GET requests.
4751
return Result(mockRepo)
4852
}
4953

50-
override fun postComplete(): Result<Unit> {
51-
Logger.debug { "MockApi: postComplete request " }
52-
receivedComplete++
53-
return Result()
54-
}
55-
5654
override fun postCommits(commitsList: List<Commit>): Result<Unit> {
5755
Logger.debug {
5856
"MockApi: postCommits request (${commitsList.size} commits)"
@@ -80,4 +78,19 @@ class MockApi( // GET requests.
8078
receivedAuthors.addAll(authorsList)
8179
return Result()
8280
}
81+
82+
override fun postProcessCreate(requestNumEntries: Int): Result<Process> {
83+
Logger.debug { "MockApi: postProcessCreate request " +
84+
"($requestNumEntries entries requested)" }
85+
receivedProcessCreate.add(
86+
Process(requestNumEntries = requestNumEntries))
87+
return Result(Process(entries = mockProcessEntries))
88+
}
89+
90+
override fun postProcess(processEntries: List<ProcessEntry>): Result<Unit> {
91+
Logger.debug { "MockApi: postProcess request (${processEntries.size} " +
92+
"entries updated)" }
93+
receivedProcess.add(Process(entries = processEntries))
94+
return Result()
95+
}
8396
}

‎src/main/kotlin/app/api/ServerApi.kt

+25-9
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import app.model.Commit
1212
import app.model.CommitGroup
1313
import app.model.Fact
1414
import app.model.FactGroup
15+
import app.model.Process
16+
import app.model.ProcessEntry
1517
import app.model.Repo
1618
import app.model.User
1719
import com.github.kittinunf.fuel.core.FuelManager
@@ -93,10 +95,6 @@ class ServerApi (private val configurator: Configurator) : Api {
9395
.body(repo.serialize())
9496
}
9597

96-
private fun createRequestPostComplete(): Request {
97-
return post("/complete").header(getContentTypeHeader())
98-
}
99-
10098
private fun createRequestPostCommits(commits: CommitGroup): Request {
10199
return post("/commits").header(getContentTypeHeader())
102100
.body(commits.serialize())
@@ -117,6 +115,16 @@ class ServerApi (private val configurator: Configurator) : Api {
117115
.body(authors.serialize())
118116
}
119117

118+
private fun createRequestPostProcessCreate(process: Process): Request {
119+
return post("/process/create").header(getContentTypeHeader())
120+
.body(process.serialize())
121+
}
122+
123+
private fun createRequestPostProcess(process: Process): Request {
124+
return post("/process").header(getContentTypeHeader())
125+
.body(process.serialize())
126+
}
127+
120128
private fun <T> makeRequest(request: Request,
121129
requestName: String,
122130
parser: (ByteArray) -> T): Result<T> {
@@ -172,11 +180,6 @@ class ServerApi (private val configurator: Configurator) : Api {
172180
{ body -> Repo(body) })
173181
}
174182

175-
override fun postComplete(): Result<Unit> {
176-
return makeRequest(createRequestPostComplete(),
177-
"postComplete", {})
178-
}
179-
180183
override fun postCommits(commitsList: List<Commit>): Result<Unit> {
181184
val commits = CommitGroup(commitsList)
182185
return makeRequest(createRequestPostCommits(commits),
@@ -198,4 +201,17 @@ class ServerApi (private val configurator: Configurator) : Api {
198201
val authors = AuthorGroup(authorsList)
199202
return makeRequest(createRequestPostAuthors(authors), "postAuthors", {})
200203
}
204+
205+
override fun postProcessCreate(requestNumEntries: Int): Result<Process> {
206+
val process = Process(requestNumEntries = requestNumEntries)
207+
return makeRequest(createRequestPostProcessCreate(process),
208+
"postProcessCreate", { body -> Process(body) })
209+
}
210+
211+
override fun postProcess(processEntries: List<ProcessEntry>): Result<Unit> {
212+
// TODO(anatoly): Restrict possible status and error codes on CS.
213+
val process = Process(entries = processEntries)
214+
return makeRequest(createRequestPostProcess(process), "postProcess", {})
215+
}
216+
201217
}

‎src/main/kotlin/app/extractors/CommonExtractor.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ class CommonExtractor : ExtractorInterface {
2929
reversedMap.put("html", listOf("html", "htm"))
3030
reversedMap.put("j", listOf("ijs"))
3131
reversedMap.put("julia", listOf("jl"))
32-
reversedMap.put("kotlin", listOf("kt"))
3332
reversedMap.put("lisp", listOf("lisp", "lsp", "l"))
3433
reversedMap.put("lua", listOf("lua"))
3534
reversedMap.put("makefile", listOf("makefile"))
@@ -74,6 +73,7 @@ class CommonExtractor : ExtractorInterface {
7473
file
7574
} else null
7675
}
76+
7777
return super.extract(files)
7878
}
7979
}

‎src/main/kotlin/app/extractors/CssExtractor.kt

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
2+
// Author: Liubov Yaronskaya ([email protected])
3+
4+
package app.extractors
5+
6+
import app.model.CommitStats
7+
import app.model.DiffFile
8+
9+
class CssExtractor : ExtractorInterface {
10+
companion object {
11+
val LANGUAGE_NAME = "css"
12+
val FILE_EXTS = listOf("css", "scss", "less", "sass")
13+
}
14+
15+
override fun extract(files: List<DiffFile>): List<CommitStats> {
16+
files.map { file -> file.language = LANGUAGE_NAME }
17+
val stats = FILE_EXTS.filter { it != "css" }.map { extension ->
18+
val result = files.filter { it.extension == extension }
19+
.fold(Pair(0, 0)) { total, file ->
20+
val currentNumAdded = file.getAllAdded()
21+
.filter { it.isNotBlank() }.size
22+
val currentNumDeleted = file.getAllDeleted()
23+
.filter { it.isNotBlank() }.size
24+
Pair(total.first + currentNumAdded,
25+
total.second + currentNumDeleted)}.toList()
26+
27+
CommitStats(numLinesAdded = result[0],
28+
numLinesDeleted = result[1],
29+
type = Extractor.TYPE_LIBRARY,
30+
tech = extension)
31+
}.filter { it.numLinesAdded > 0 || it.numLinesDeleted > 0 }
32+
33+
return stats + super.extract(files)
34+
}
35+
}

‎src/main/kotlin/app/extractors/Extractor.kt

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class Extractor : ExtractorInterface {
3131
in ObjectiveCExtractor.FILE_EXTS -> ObjectiveCExtractor()
3232
in SwiftExtractor.FILE_EXTS -> SwiftExtractor()
3333
in KotlinExtractor.FILE_EXTS -> KotlinExtractor()
34+
in CssExtractor.FILE_EXTS -> CssExtractor()
3435
else -> CommonExtractor()
3536
}
3637
}

‎src/main/kotlin/app/hashers/RepoHasher.kt

+20-4
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import app.api.Api
99
import app.config.Configurator
1010
import app.model.Author
1111
import app.model.LocalRepo
12+
import app.model.ProcessEntry
1213
import app.model.Repo
1314
import app.utils.FileHelper.toPath
1415
import app.utils.HashingException
@@ -19,7 +20,8 @@ import java.io.IOException
1920
import kotlin.collections.HashSet
2021

2122
class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
22-
private val configurator: Configurator) {
23+
private val configurator: Configurator,
24+
private val processEntryId: Int? = null) {
2325
var serverRepo: Repo = Repo()
2426

2527
init {
@@ -32,6 +34,7 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
3234
Logger.info { "Hashing of repo started" }
3335
val git = loadGit(localRepo.path)
3436
try {
37+
updateProcess(processEntryId, Api.PROCESS_STATUS_START)
3538
val (rehashes, authors) = CommitCrawler.fetchRehashesAndAuthors(git)
3639

3740
localRepo.parseGitConfig(git.repository.config)
@@ -88,11 +91,13 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
8891
if (errors.isNotEmpty()) {
8992
throw HashingException(errors)
9093
}
91-
9294
Logger.info(Logger.Events.HASHING_REPO_SUCCESS)
9395
{ "Hashing repo completed" }
94-
}
95-
finally {
96+
updateProcess(processEntryId, Api.PROCESS_STATUS_COMPLETE)
97+
} catch (e: Throwable) {
98+
updateProcess(processEntryId, Api.PROCESS_STATUS_FAIL)
99+
throw e
100+
} finally {
96101
closeGit(git)
97102
}
98103
}
@@ -143,4 +148,15 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
143148

144149
return knownEmails.filter { emails.contains(it) }.toHashSet()
145150
}
151+
152+
private fun updateProcess(processEntryId: Int?, status: Int,
153+
errorCode: Int = 0) {
154+
if (processEntryId == null) {
155+
return
156+
}
157+
158+
val processEntry = ProcessEntry(id = processEntryId, status = status,
159+
errorCode = errorCode)
160+
api.postProcess(listOf(processEntry)).onErrorThrow()
161+
}
146162
}

‎src/main/kotlin/app/model/Process.kt

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package app.model
2+
3+
import app.Protos
4+
import com.google.protobuf.InvalidProtocolBufferException
5+
import java.security.InvalidParameterException
6+
7+
/**
8+
* Used to describe processing of multiple repos.
9+
*/
10+
data class Process(
11+
var id: Int = 0,
12+
var requestNumEntries: Int = 0,
13+
var entries: List<ProcessEntry> = mutableListOf()
14+
) {
15+
@Throws(InvalidParameterException::class)
16+
constructor(proto: Protos.Process) : this() {
17+
id = proto.id
18+
requestNumEntries = proto.requestNumEntries
19+
entries = proto.entriesList.map { ProcessEntry(it) }
20+
}
21+
22+
@Throws(InvalidProtocolBufferException::class)
23+
constructor(bytes: ByteArray) : this(Protos.Process.parseFrom(bytes))
24+
25+
constructor(serialized: String) : this(serialized.toByteArray())
26+
27+
fun getProto(): Protos.Process {
28+
return Protos.Process.newBuilder()
29+
.setId(id)
30+
.setRequestNumEntries(requestNumEntries)
31+
.addAllEntries(entries.map { it.getProto() })
32+
.build()
33+
}
34+
35+
fun serialize(): ByteArray {
36+
return getProto().toByteArray()
37+
}
38+
}

‎src/main/kotlin/app/model/ProcessEntry.kt

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package app.model
2+
3+
import app.Protos
4+
import com.google.protobuf.InvalidProtocolBufferException
5+
import java.security.InvalidParameterException
6+
7+
/**
8+
* Used to describe processing of a single repo.
9+
*/
10+
data class ProcessEntry(
11+
var id: Int = 0,
12+
var status: Int = 0,
13+
var errorCode: Int = 0
14+
) {
15+
@Throws(InvalidParameterException::class)
16+
constructor(proto: Protos.ProcessEntry) : this() {
17+
id = proto.id
18+
status = proto.status
19+
errorCode = proto.errorCode
20+
}
21+
22+
@Throws(InvalidProtocolBufferException::class)
23+
constructor(bytes: ByteArray) : this(Protos.ProcessEntry.parseFrom(bytes))
24+
25+
constructor(serialized: String) : this(serialized.toByteArray())
26+
27+
fun getProto(): Protos.ProcessEntry {
28+
return Protos.ProcessEntry.newBuilder()
29+
.setId(id)
30+
.setStatus(status)
31+
.setErrorCode(errorCode)
32+
.build()
33+
}
34+
35+
fun serialize(): ByteArray {
36+
return getProto().toByteArray()
37+
}
38+
}

0 commit comments

Comments
 (0)