render-diff endpoint impl

Author: akvlad

patterns/patterns_bin/src/pattern_reg.rs

@@ -0,0 +1,45 @@
+use crate::pattern::Pattern;
+use uuid::Uuid;
+
+pub struct PatternRegistry {
+    patterns: Vec<Pattern>,
+}
+
+impl PatternRegistry {
+    pub const fn new() -> PatternRegistry {
+        PatternRegistry { patterns: Vec::new() }
+    }
+
+    pub fn find_pattern(&mut self, str_text: &Vec<String>, i_text: &Vec<u64>, sample: String) -> &Pattern {
+        let mut idx: i32 = -1;
+        let mut mtc = 0;
+        for i in 0..self.patterns.len() {
+            mtc = self.patterns[i].match_text(&i_text);
+            if mtc == -1 || mtc > self.patterns[i].fluct {
+                continue;
+            }
+            idx = i as i32;
+            break;
+        }
+
+        if idx == -1 {
+            let pattern = Pattern::new(Uuid::new_v4().to_string(), &i_text, &str_text, sample);
+            self.patterns.push(pattern);
+            idx = (self.patterns.len() - 1) as i32;
+        } else if mtc != 0 {
+            self.patterns[idx as usize].adjust_pattern(&i_text);
+        }
+        return &self.patterns[idx as usize];
+    }
+
+    pub fn to_string(&self) -> String {
+        let mut s = String::new();
+        for i in 0..self.patterns.len() {
+            s += self.patterns[i].to_string().as_str();
+            s += "\n";
+        }
+        return s
+    }
+}
+
+pub static mut REGISTRY: PatternRegistry = PatternRegistry::new();

patterns/patterns_bin/src/tokens.rs

@@ -0,0 +1,45 @@
+use regex::{Regex, CaptureMatches, Match};
+
+/*pub fn tokenize(re: &Regex, text: &str) -> CaptureMatches {
+    return re.captures_iter(text);
+}*/
+
+pub struct Tokenizer<'a> {
+    text: String,
+    pos: usize,
+    re: Regex,
+    iter: Option<CaptureMatches<'a, 'a>>
+}
+
+impl Tokenizer<'_> {
+    pub fn new<'a>(text: &'a str) -> Tokenizer<'a> {
+        let mut res = Tokenizer {
+            text: text.to_string(),
+            pos: 0,
+            re: Regex::new(r"([\p{L}_]+|[\d.]+|[^\p{L}_\d.]+)\s*").unwrap(),
+            iter: None
+        };
+        res
+    }
+}
+
+impl Iterator for Tokenizer<'_> {
+    type Item = String;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        None
+        /*let cap: Option<Match> = None;
+        if let Some(c) = cap {
+            self.pos += c.get(0).unwrap().end();
+            Some(c.get(0).unwrap().as_str().to_string())
+        } else {
+            None
+        }*/
+    }
+}
+
+#[test]
+fn test_tokenizer() {
+    let text = "Hello, world! 123";
+    let mut tokenizer = Tokenizer::new(text);
+}

pyroscope/flamebearer.d.ts

@@ -52,3 +52,16 @@ export interface heatmap {
     minDepth: uint64,
     maxDepth: uint64
 }
+
+export interface level {
+    values: number[]
+}
+
+export interface flamegraphDiff {
+    name: string[],
+    levels: level[],
+    total: int64,
+    maxSelf: int64,
+    leftTicks: int64,
+    rightTicks: int64
+}

pyroscope/merge_stack_traces.js

@@ -22,7 +22,9 @@ const sqlWithReference = (ref) => {
 
 let ctxIdx = 0
 
-const mergeStackTraces = async (typeRegex, sel, fromTimeSec, toTimeSec, log) => {
+const newCtxIdx = () => ++ctxIdx
+
+const importStackTraces = async (typeRegex, sel, fromTimeSec, toTimeSec, log, _ctxIdx, save) => {
   const dist = clusterName ? '_dist' : ''
   const v2 = checkVersion('profiles_v2', (fromTimeSec - 3600) * 1000)
   const serviceNameSelector = serviceNameSelectorQuery(sel)
@@ -127,30 +129,36 @@ const mergeStackTraces = async (typeRegex, sel, fromTimeSec, toTimeSec, log) =>
   const binData = Uint8Array.from(profiles.data)
   log.debug(`selectMergeStacktraces: profiles downloaded: ${binData.length / 1025}kB in ${Date.now() - start}ms`)
   require('./pprof-bin/pkg/pprof_bin').init_panic_hook()
-  const _ctxIdx = ++ctxIdx
   const [legacyLen, shift] = readULeb32(binData, 0)
   let ofs = shift
-  try {
-    let mergePprofLat = BigInt(0)
-    for (let i = 0; i < legacyLen; i++) {
-      const [profLen, shift] = readULeb32(binData, ofs)
-      ofs += shift
-      start = process.hrtime?.bigint ? process.hrtime.bigint() : BigInt(0)
-      pprofBin.merge_prof(_ctxIdx,
-        Uint8Array.from(profiles.data.slice(ofs, ofs + profLen)),
-        `${typeRegex.sampleType}:${typeRegex.sampleUnit}`)
-      mergePprofLat += (process.hrtime?.bigint ? process.hrtime.bigint() : BigInt(0)) - start
-      ofs += profLen
-    }
-    start = process.hrtime?.bigint ? process.hrtime.bigint() : BigInt(0)
-    pprofBin.merge_tree(_ctxIdx, Uint8Array.from(profiles.data.slice(ofs)),
-      typeRegex.sampleType + ':' + typeRegex.sampleUnit)
-    const mergeTreeLat = (process.hrtime?.bigint ? process.hrtime.bigint() : BigInt(0)) - start
+  let mergePprofLat = BigInt(0)
+  for (let i = 0; i < legacyLen; i++) {
+    const [profLen, shift] = readULeb32(binData, ofs)
+    ofs += shift
     start = process.hrtime?.bigint ? process.hrtime.bigint() : BigInt(0)
+    pprofBin.merge_prof(_ctxIdx,
+      Uint8Array.from(profiles.data.slice(ofs, ofs + profLen)),
+      `${typeRegex.sampleType}:${typeRegex.sampleUnit}`)
+    mergePprofLat += (process.hrtime?.bigint ? process.hrtime.bigint() : BigInt(0)) - start
+    ofs += profLen
+  }
+  start = process.hrtime?.bigint ? process.hrtime.bigint() : BigInt(0)
+  save && require('fs').writeFileSync(`/home/hromozeka/QXIP/qryn/data.${Date.now()}.bin`,
+    Buffer.from(Uint8Array.from(profiles.data.slice(ofs))))
+  pprofBin.merge_tree(_ctxIdx, Uint8Array.from(profiles.data.slice(ofs)),
+    typeRegex.sampleType + ':' + typeRegex.sampleUnit)
+  const mergeTreeLat = (process.hrtime?.bigint ? process.hrtime.bigint() : BigInt(0)) - start
+  log.debug(`merge_pprof: ${mergePprofLat / BigInt(1000000)}ms`)
+  log.debug(`merge_tree: ${mergeTreeLat / BigInt(1000000)}ms`)
+}
+
+const mergeStackTraces = async (typeRegex, sel, fromTimeSec, toTimeSec, log) => {
+  const _ctxIdx = newCtxIdx()
+  try {
+    await importStackTraces(typeRegex, sel, fromTimeSec, toTimeSec, log, _ctxIdx)
+    const start = process.hrtime?.bigint ? process.hrtime.bigint() : BigInt(0)
     const resp = pprofBin.export_tree(_ctxIdx, typeRegex.sampleType + ':' + typeRegex.sampleUnit)
     const exportTreeLat = (process.hrtime?.bigint ? process.hrtime.bigint() : BigInt(0)) - start
-    log.debug(`merge_pprof: ${mergePprofLat / BigInt(1000000)}ms`)
-    log.debug(`merge_tree: ${mergeTreeLat / BigInt(1000000)}ms`)
     log.debug(`export_tree: ${exportTreeLat / BigInt(1000000)}ms`)
     return Buffer.from(resp)
   } finally {
@@ -159,5 +167,7 @@ const mergeStackTraces = async (typeRegex, sel, fromTimeSec, toTimeSec, log) =>
 }
 
 module.exports = {
-  mergeStackTraces
-}
+  mergeStackTraces,
+  importStackTraces,
+  newCtxIdx
+}

pyroscope/pprof-bin/pkg/pprof_bin.d.ts

@@ -13,6 +13,13 @@ export function merge_prof(id: number, bytes: Uint8Array, sample_type: string):
 */
 export function merge_tree(id: number, bytes: Uint8Array, sample_type: string): void;
 /**
+* @param {number} id1
+* @param {number} id2
+* @param {string} sample_type
+* @returns {Uint8Array}
+*/
+export function diff_tree(id1: number, id2: number, sample_type: string): Uint8Array;
+/**
 * @param {number} id
 * @param {string} sample_type
 * @returns {Uint8Array}

pyroscope/pprof-bin/pkg/pprof_bin.js

@@ -133,6 +133,28 @@ function getArrayU8FromWasm0(ptr, len) {
     ptr = ptr >>> 0;
     return getUint8Memory0().subarray(ptr / 1, ptr / 1 + len);
 }
+/**
+* @param {number} id1
+* @param {number} id2
+* @param {string} sample_type
+* @returns {Uint8Array}
+*/
+module.exports.diff_tree = function(id1, id2, sample_type) {
+    try {
+        const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+        const ptr0 = passStringToWasm0(sample_type, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.diff_tree(retptr, id1, id2, ptr0, len0);
+        var r0 = getInt32Memory0()[retptr / 4 + 0];
+        var r1 = getInt32Memory0()[retptr / 4 + 1];
+        var v2 = getArrayU8FromWasm0(r0, r1).slice();
+        wasm.__wbindgen_free(r0, r1 * 1, 1);
+        return v2;
+    } finally {
+        wasm.__wbindgen_add_to_stack_pointer(16);
+    }
+};
+
 /**
 * @param {number} id
 * @param {string} sample_type

pyroscope/pprof-bin/pkg/pprof_bin_bg.wasm

@@ -3,6 +3,7 @@
 export const memory: WebAssembly.Memory;
 export function merge_prof(a: number, b: number, c: number, d: number, e: number): void;
 export function merge_tree(a: number, b: number, c: number, d: number, e: number): void;
+export function diff_tree(a: number, b: number, c: number, d: number, e: number): void;
 export function export_tree(a: number, b: number, c: number, d: number): void;
 export function export_trees_pprof(a: number, b: number, c: number): void;
 export function drop_tree(a: number): void;