Skip to content

Commit dc838e1

Browse files
nobodyiamnobodyiam
committed
automatically update docs
1 parent b4a4dc8 commit dc838e1

File tree

9 files changed

+510
-16
lines changed

9 files changed

+510
-16
lines changed

‎src/main/java/com/apolloconfig/apollo/ai/qabot/config/MarkdownFilesConfig.java

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package com.apolloconfig.apollo.ai.qabot.config;
2+
3+
import com.google.common.collect.Lists;
4+
import java.util.List;
5+
import org.springframework.boot.context.properties.ConfigurationProperties;
6+
import org.springframework.stereotype.Component;
7+
8+
@ConfigurationProperties(prefix = "markdown.files")
9+
@Component
10+
public class MarkdownFilesConfig {
11+
12+
private String location;
13+
private List<String> roots = Lists.newLinkedList();
14+
15+
public List<String> getRoots() {
16+
return roots;
17+
}
18+
19+
public String getLocation() {
20+
return location;
21+
}
22+
23+
public void setLocation(String location) {
24+
this.location = location;
25+
}
26+
27+
public void setRoots(List<String> roots) {
28+
this.roots = roots;
29+
}
30+
}

‎src/main/java/com/apolloconfig/apollo/ai/qabot/config/MarkdownProcessorRetryConfig.java

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package com.apolloconfig.apollo.ai.qabot.config;
2+
3+
import org.springframework.boot.context.properties.ConfigurationProperties;
4+
import org.springframework.stereotype.Component;
5+
6+
@ConfigurationProperties(prefix = "markdown.processor.retry")
7+
@Component
8+
public class MarkdownProcessorRetryConfig {
9+
10+
private long delay;
11+
private double multiplier;
12+
private long maxDelay;
13+
private long maxElapsedTime;
14+
15+
public long getDelay() {
16+
return delay;
17+
}
18+
19+
public void setDelay(long delay) {
20+
this.delay = delay;
21+
}
22+
23+
public double getMultiplier() {
24+
return multiplier;
25+
}
26+
27+
public void setMultiplier(double multiplier) {
28+
this.multiplier = multiplier;
29+
}
30+
31+
public long getMaxDelay() {
32+
return maxDelay;
33+
}
34+
35+
public void setMaxDelay(long maxDelay) {
36+
this.maxDelay = maxDelay;
37+
}
38+
39+
public long getMaxElapsedTime() {
40+
return maxElapsedTime;
41+
}
42+
43+
public void setMaxElapsedTime(long maxElapsedTime) {
44+
this.maxElapsedTime = maxElapsedTime;
45+
}
46+
}

‎src/main/java/com/apolloconfig/apollo/ai/qabot/controller/QAWithAssistantsController.java

Lines changed: 56 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,22 @@
22

33
import com.apolloconfig.apollo.ai.qabot.entity.Answer;
44
import com.apolloconfig.apollo.ai.qabot.openai.OpenAiAssistantsService;
5+
import com.fasterxml.jackson.core.JsonProcessingException;
56
import com.fasterxml.jackson.databind.ObjectMapper;
7+
import com.google.common.base.Function;
68
import com.google.common.base.Strings;
79
import com.theokanning.openai.assistants.StreamEvent;
10+
import com.theokanning.openai.assistants.message.Message;
811
import com.theokanning.openai.assistants.message.content.Annotation;
912
import com.theokanning.openai.assistants.message.content.MessageDelta;
1013
import com.theokanning.openai.assistants.message.content.Text;
1114
import com.theokanning.openai.service.assistant_stream.AssistantSSE;
1215
import io.reactivex.Flowable;
16+
import java.util.Collections;
17+
import java.util.List;
18+
import java.util.Set;
19+
import java.util.stream.Collectors;
20+
import org.jetbrains.annotations.NotNull;
1321
import org.slf4j.Logger;
1422
import org.slf4j.LoggerFactory;
1523
import org.springframework.http.MediaType;
@@ -67,24 +75,60 @@ private Flux<Answer> doQA(String threadId, String question) {
6775
Flowable<AssistantSSE> result = aiService.getAssistantMessage(threadId, question);
6876

6977
Flux<Answer> flux = Flux.from(result.filter(
70-
assistantSSE -> assistantSSE.getEvent() == StreamEvent.THREAD_MESSAGE_DELTA)
78+
assistantSSE -> assistantSSE.getEvent() == StreamEvent.THREAD_MESSAGE_DELTA
79+
|| assistantSSE.getEvent() == StreamEvent.THREAD_MESSAGE_COMPLETED)
7180
.map(assistantSSE -> {
7281
if (LOGGER.isDebugEnabled()) {
7382
LOGGER.debug("event: {}, data: {}", assistantSSE.getEvent(),
7483
assistantSSE.getData());
7584
}
76-
MessageDelta message = objectMapper.readValue(assistantSSE.getData(),
77-
MessageDelta.class);
78-
Text text = message.getDelta().getContent().get(0).getText();
79-
String value = text.getValue();
80-
if (!CollectionUtils.isEmpty(text.getAnnotations())) {
81-
for (Annotation annotation : text.getAnnotations()) {
82-
value = value.replace(annotation.getText(), "");
83-
}
85+
86+
// fetch the related files
87+
if (assistantSSE.getEvent() == StreamEvent.THREAD_MESSAGE_COMPLETED) {
88+
return getAnswerFromMessage(assistantSSE);
8489
}
85-
return new Answer(value, "");
86-
}));
8790

88-
return flux.concatWith(Flux.just(new Answer(END_SYMBOL, threadId)));
91+
return getAnswerFromMessageDelta(assistantSSE);
92+
})).onErrorReturn(Answer.ERROR);
93+
94+
return flux.concatWith(Flux.just(new Answer(END_SYMBOL, threadId, Collections.emptySet())));
95+
}
96+
97+
private @NotNull Answer getAnswerFromMessage(AssistantSSE assistantSSE)
98+
throws JsonProcessingException {
99+
Message message = objectMapper.readValue(assistantSSE.getData(), Message.class);
100+
List<Annotation> annotations = message.getContent().get(0).getText().getAnnotations();
101+
if (CollectionUtils.isEmpty(annotations)) {
102+
return Answer.EMPTY;
103+
}
104+
Set<String> relatedFiles = annotations.stream()
105+
.filter(annotation -> annotation.getType().equals("file_citation")).map(
106+
(Function<Annotation, String>) input -> {
107+
try {
108+
String fileName = aiService.getFileName(input.getFileCitation().getFileId());
109+
if (fileName.endsWith(".md")) {
110+
fileName = fileName.substring(0, fileName.length() - 3);
111+
}
112+
return fileName;
113+
114+
} catch (Throwable ex) {
115+
LOGGER.error("Error while fetching file name", ex);
116+
return "";
117+
}
118+
}).collect(Collectors.toSet());
119+
return new Answer("", "", relatedFiles);
120+
}
121+
122+
private @NotNull Answer getAnswerFromMessageDelta(AssistantSSE assistantSSE)
123+
throws JsonProcessingException {
124+
MessageDelta message = objectMapper.readValue(assistantSSE.getData(), MessageDelta.class);
125+
Text text = message.getDelta().getContent().get(0).getText();
126+
String value = text.getValue();
127+
if (!CollectionUtils.isEmpty(text.getAnnotations())) {
128+
for (Annotation annotation : text.getAnnotations()) {
129+
value = value.replace(annotation.getText(), "");
130+
}
131+
}
132+
return new Answer(value, "", Collections.emptySet());
89133
}
90134
}

‎src/main/java/com/apolloconfig/apollo/ai/qabot/entity/Answer.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
package com.apolloconfig.apollo.ai.qabot.entity;
22

3-
public record Answer(String answer, String threadId) {
3+
import java.util.Collections;
4+
import java.util.Set;
45

5-
public static final Answer EMPTY = new Answer("", "");
6+
public record Answer(String answer, String threadId, Set<String> relatedFiles) {
7+
8+
public static final Answer EMPTY = new Answer("", "", Collections.emptySet());
69
public static final Answer ERROR = new Answer(
7-
"Sorry, I can't answer your question right now. Please try again later.", "");
10+
"Sorry, I can't answer your question right now. Please try again later.", "",
11+
Collections.emptySet());
812
}

‎src/main/java/com/apolloconfig/apollo/ai/qabot/markdown/MarkdownProcessor.java

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
package com.apolloconfig.apollo.ai.qabot.markdown;
2+
3+
import com.apolloconfig.apollo.ai.qabot.config.MarkdownFilesConfig;
4+
import com.apolloconfig.apollo.ai.qabot.config.MarkdownProcessorRetryConfig;
5+
import com.apolloconfig.apollo.ai.qabot.openai.OpenAiAssistantsService;
6+
import com.google.common.collect.Maps;
7+
import java.io.IOException;
8+
import java.nio.file.Files;
9+
import java.nio.file.Path;
10+
import java.nio.file.Paths;
11+
import java.security.MessageDigest;
12+
import java.security.NoSuchAlgorithmException;
13+
import java.util.ArrayList;
14+
import java.util.List;
15+
import java.util.Map;
16+
import java.util.Objects;
17+
import java.util.function.Function;
18+
import java.util.stream.Stream;
19+
import org.slf4j.Logger;
20+
import org.slf4j.LoggerFactory;
21+
import org.springframework.stereotype.Service;
22+
import org.springframework.util.backoff.BackOff;
23+
import org.springframework.util.backoff.BackOffExecution;
24+
import org.springframework.util.backoff.ExponentialBackOff;
25+
import retrofit2.HttpException;
26+
27+
@Service
28+
public class MarkdownProcessor {
29+
30+
private static final Logger LOGGER = LoggerFactory.getLogger(MarkdownProcessor.class);
31+
32+
private final OpenAiAssistantsService aiService;
33+
private final MarkdownFilesConfig markdownFilesConfig;
34+
private final MarkdownProcessorRetryConfig markdownProcessorRetryConfig;
35+
private final BackOff backOff;
36+
private final Map<String, String> fileHashValues;
37+
private final Map<String, String> fileIds;
38+
39+
public MarkdownProcessor(OpenAiAssistantsService aiService,
40+
MarkdownFilesConfig markdownFilesConfig,
41+
MarkdownProcessorRetryConfig markdownProcessorRetryConfig) {
42+
this.aiService = aiService;
43+
this.markdownFilesConfig = markdownFilesConfig;
44+
this.markdownProcessorRetryConfig = markdownProcessorRetryConfig;
45+
this.backOff = initializeBackOff();
46+
this.fileHashValues = Maps.newConcurrentMap();
47+
this.fileIds = this.aiService.getVectorStoreFileIds();
48+
}
49+
50+
private BackOff initializeBackOff() {
51+
ExponentialBackOff exponentialBackOff = new ExponentialBackOff();
52+
exponentialBackOff.setInitialInterval(markdownProcessorRetryConfig.getDelay());
53+
exponentialBackOff.setMultiplier(markdownProcessorRetryConfig.getMultiplier());
54+
exponentialBackOff.setMaxInterval(markdownProcessorRetryConfig.getMaxDelay());
55+
exponentialBackOff.setMaxElapsedTime(markdownProcessorRetryConfig.getMaxElapsedTime());
56+
57+
return exponentialBackOff;
58+
}
59+
60+
public void initialize(String location) {
61+
processWithAction(location, path -> {
62+
try {
63+
String markdownContent = Files.readString(path);
64+
fileHashValues.put(getMarkdownFileRoots(path), computeHash(markdownContent));
65+
} catch (IOException e) {
66+
throw new RuntimeException(e);
67+
}
68+
return true;
69+
});
70+
}
71+
72+
public List<String> loadAndProcessFiles(String location) {
73+
return this.processWithAction(location, this::processFileWithRetry);
74+
}
75+
76+
private List<String> processWithAction(String location, Function<Path, Boolean> action) {
77+
List<String> updatedFiles = new ArrayList<>();
78+
Path mdDirectory = Paths.get(location);
79+
try (Stream<Path> paths = Files.walk(mdDirectory)) {
80+
paths
81+
.filter(Files::isRegularFile)
82+
.filter(path -> path.toString().endsWith(".md"))
83+
.forEach(mdFile -> {
84+
try {
85+
boolean result = action.apply(mdFile);
86+
if (result) {
87+
updatedFiles.add(mdFile.toAbsolutePath().toString());
88+
}
89+
} catch (Throwable e) {
90+
LOGGER.error("Error processing file {}", mdFile.getFileName(), e);
91+
}
92+
});
93+
} catch (Throwable e) {
94+
LOGGER.error("Error reading files from location {}", location, e);
95+
}
96+
97+
return updatedFiles;
98+
}
99+
100+
private boolean processFileWithRetry(Path mdFile) {
101+
BackOffExecution backOffExecution = backOff.start();
102+
while (!Thread.currentThread().isInterrupted()) {
103+
try {
104+
return processFile(mdFile);
105+
} catch (HttpException exception) {
106+
if (exception.code() == 429) {
107+
long sleepTime = backOffExecution.nextBackOff();
108+
109+
if (sleepTime == BackOffExecution.STOP) {
110+
LOGGER.error("Retry limit exceeded. Stopping");
111+
break;
112+
}
113+
114+
LOGGER.warn("OpenAI API rate limit exceeded. Retrying in {} ms", sleepTime);
115+
116+
try {
117+
Thread.sleep(sleepTime);
118+
} catch (InterruptedException e) {
119+
LOGGER.error("Interrupted while waiting for retry", e);
120+
Thread.currentThread().interrupt();
121+
break;
122+
}
123+
} else {
124+
throw exception;
125+
}
126+
} catch (Throwable e) {
127+
throw new RuntimeException(e);
128+
}
129+
}
130+
131+
return false;
132+
}
133+
134+
boolean processFile(Path mdFile) throws IOException {
135+
String fileRoot = getMarkdownFileRoots(mdFile);
136+
137+
String markdownContent = Files.readString(mdFile);
138+
String hashValue = computeHash(markdownContent);
139+
140+
String fileHashValue = this.fileHashValues.get(fileRoot);
141+
142+
if (Objects.equals(hashValue, fileHashValue)) {
143+
return false;
144+
}
145+
146+
LOGGER.debug("File {} has changed", mdFile.getFileName());
147+
148+
String fileId = this.fileIds.get(fileRoot);
149+
if (fileId != null) {
150+
this.aiService.deleteVectorStoreFile(fileId);
151+
this.fileIds.remove(fileRoot);
152+
}
153+
154+
String newFileId = this.aiService.createVectorStoreFile(fileRoot, markdownContent);
155+
this.fileIds.put(fileRoot, newFileId);
156+
this.fileHashValues.put(fileRoot, hashValue);
157+
158+
return true;
159+
}
160+
161+
private String getMarkdownFileRoots(Path mdFile) {
162+
String fullPath = mdFile.toAbsolutePath().toString();
163+
for (String root : markdownFilesConfig.getRoots()) {
164+
if (fullPath.contains(root)) {
165+
fullPath = fullPath.substring(fullPath.indexOf(root));
166+
}
167+
}
168+
169+
return fullPath;
170+
}
171+
172+
private String computeHash(String input) {
173+
try {
174+
MessageDigest md = MessageDigest.getInstance("SHA-256");
175+
byte[] hash = md.digest(input.getBytes());
176+
StringBuilder hexString = new StringBuilder(2 * hash.length);
177+
for (byte b : hash) {
178+
String hex = Integer.toHexString(0xff & b);
179+
if (hex.length() == 1) {
180+
hexString.append('0');
181+
}
182+
hexString.append(hex);
183+
}
184+
return hexString.toString();
185+
} catch (NoSuchAlgorithmException e) {
186+
throw new RuntimeException(e);
187+
}
188+
}
189+
190+
}

0 commit comments

Comments
 (0)