Initial commit

README.md

+# TraceVAE
+This is the source code for "Unsupervised Anomaly Detection on Microservice Traces through Graph VAE".
+
+## Usage
+1. `pip3 install -r requirements.txt`.
+2. Convert the dataset with `python3 -m tracegnn.cli.data_process preprocess -i [input_path] -o [dataset_path]`. The sample dataset is under `sample_dataset`. (Note: This sample dataset only shows data format and usage, and cannot be used to evaluate model performance. Please replace it with your dataset.)
+sample:
+```
+python3 -m tracegnn.cli.data_process preprocess -i sample_dataset -o sample_dataset
+```
+3. Train the model with `bash train.sh [dataset_path]`:
+```
+bash train.sh sample_dataset
+```
+4. Evaluate the model with `bash teset.sh [model_path] [dataset_path]`. The default model path is under `results/train/models/final.pt`:
+```
+bash test.sh results/train/models/final.pt sample_dataset
+```

requirements.txt

+git+https://gitee.com/haowen-xu/ml-essentials
+git+https://gitee.com/haowen-xu/tensorkit
+git+https://gitee.com/mirrors/ZhuSuan.git
+click
+jinja2
+networkx
+numpy
+pandas
+PyYAML
+python-snappy
+scikit-learn
+seaborn
+semver
+natsort
+imageio
+fs
+lazy_object_proxy
+kafka-python
+tqdm
+loguru
+tensorboard==1.15
+numba
+
+-f https://download.pytorch.org/whl/cu116/torch_stable.html
+torch==1.12.0+cu116
+
+-f https://data.dgl.ai/wheels/repo.html
+dgl-cu116
+
+-f https://data.pyg.org/whl/torch-1.12.0+cu116.html
+torch-scatter
+torch-sparse
+torch-cluster
+torch-spline-conv
+torch-geometric

sample_dataset/id_manager/status_id.yml

+? ''
+: 0
+'0': 1
+'1': 2
+'400': 3
+'401': 4
+'403': 5
+'404': 6
+'423': 7
+'429': 8
+'500': 9
+'503': 10
+'504': 11
+CompletionException: 12
+HystrixRuntimeException: 13
+ProcessingException: 14
+SOAERROR_1006: 15
+SOAERROR_2017: 16
+SOAERROR_2026: 17
+SOAERROR_2028: 18
+UNFINISHED: 19

sample_dataset/test.csv

+traceIdHigh,traceIdLow,parentSpanId,spanId,startTime,duration,nanosecond,DBhash,status,operationName,serviceName,nodeLatencyLabel,graphLatencyLabel,graphStructureLabel
+1596357529536981502,4448694259929862755,0,-1990250972,2022-05-02 18:10:09,4188,102000000,0,0,1413,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,-281558527,2022-05-02 18:10:09,3913,347000000,0,0,1399,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,-335450563,2022-05-02 18:10:09,74,256000000,0,0,1400,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,159928739,2022-05-02 18:10:09,14,166000000,0,0,1402,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,1875814534,2022-05-02 18:10:09,20,180000000,0,0,1402,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,-1741606249,2022-05-02 18:10:09,20,250000000,0,0,1402,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,-1617443693,2022-05-02 18:10:09,12,268000000,0,0,1402,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,-278055518,2022-05-02 18:10:09,12,328000000,0,0,1402,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,-370879640,2022-05-02 18:10:09,11,349000000,0,0,1402,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,2071447605,2022-05-02 18:10:13,15,265000000,0,0,1402,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,-1819031907,2022-05-02 18:10:13,12,278000000,0,0,1402,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,1842451354,2022-05-02 18:10:13,13,287000000,0,0,1402,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,1935560096,2022-05-02 18:10:13,15,295000000,0,0,1402,310,0,0,1
+1596357529536981502,4448694259929862755,-1990250972,-2003257890,2022-05-02 18:10:14,14,306000000,0,500,1412,310,0,0,1
+1596357529536981502,4448694259929862755,-281558527,-700384075,2022-05-02 18:10:09,3907,353000000,0,0,124,30,0,0,1
+1596357529536981502,4448694259929862755,-335450563,-21249935,2022-05-02 18:10:09,63,267000000,0,0,249,42,0,0,1
+1596357529536981502,4448694259929862755,-2003257890,1900376561,2022-05-02 18:10:14,2,318000000,0,500,1370,305,0,0,1
+1596357529536981502,4448694259929862755,-700384075,-803543939,2022-05-02 18:10:09,23,367000000,0,0,113,30,0,0,1
+1596357529536981502,4448694259929862755,-700384075,-736459037,2022-05-02 18:10:09,19,371000000,0,0,113,30,0,0,1
+1596357529536981502,4448694259929862755,-700384075,-1183351125,2022-05-02 18:10:11,16,34000000,0,0,113,30,0,0,1
+1596357529536981502,4448694259929862755,-700384075,350862234,2022-05-02 18:10:12,10,580000000,0,0,113,30,0,0,1
+1596357529536981502,4448694259929862755,-700384075,1682015471,2022-05-02 18:10:13,11,169000000,0,0,113,30,0,0,1
+1596357529536981502,4448694259929862755,-700384075,2040964166,2022-05-02 18:10:13,11,259000000,0,0,113,30,0,0,1
+1596357529536981502,4448694259929862755,-700384075,1762151399,2022-05-02 18:10:10,955,85000000,0,0,117,30,0,0,1
+1596357529536981502,4448694259929862755,-700384075,-334971362,2022-05-02 18:10:12,583,577000000,0,0,117,30,0,0,1
+1596357529536981502,4448694259929862755,-700384075,-15543737,2022-05-02 18:10:09,51,379000000,0,0,119,30,0,0,1
+1596357529536981502,4448694259929862755,-700384075,583885210,2022-05-02 18:10:14,11,269000000,0,0,121,30,0,0,1
+1596357529536981502,4448694259929862755,1762151399,681721006,2022-05-02 18:10:10,840,160000000,0,0,1270,281,0,0,1
+1596357529536981502,4448694259929862755,-334971362,1067395986,2022-05-02 18:10:12,479,611000000,0,0,1270,281,0,0,1
+1596357529536981502,4448694259929862755,-15543737,408810147,2022-05-02 18:10:09,33,397000000,0,0,1277,284,0,0,1
+1596357529536981502,4448694259929862755,583885210,-304841780,2022-05-02 18:10:14,3,277000000,0,0,1369,305,0,0,1
+5662566985137355690,7477298637177050925,0,-491131385,2022-05-02 17:47:41,5176,914000000,0,0,1393,308,0,1,0
+5662566985137355690,7477298637177050925,-491131385,-652820602,2022-05-02 17:47:41,14,976000000,0,0,1378,308,0,1,0
+5662566985137355690,7477298637177050925,-491131385,-1333335415,2022-05-02 17:47:47,11,69000000,0,0,1380,308,0,1,0
+5662566985137355690,7477298637177050925,-491131385,703056482,2022-05-02 17:47:41,10,950000000,0,0,1381,308,0,1,0
+5662566985137355690,7477298637177050925,-491131385,687971319,2022-05-02 17:47:47,7,83000000,0,0,1386,308,1,1,0
+5662566985137355690,7477298637177050925,-491131385,1792075409,2022-05-02 17:47:41,7,963000000,0,0,1389,308,0,1,0
+5662566985137355690,7477298637177050925,-491131385,48140695,2022-05-02 17:47:47,26,14000000,0,0,1391,308,0,1,0
+5662566985137355690,7477298637177050925,-652820602,1379876670,2022-05-02 17:47:41,11,979000000,0,0,310,57,0,1,0
+5662566985137355690,7477298637177050925,703056482,-699583531,2022-05-02 17:47:41,5,945000000,0,0,356,72,0,1,0
+5662566985137355690,7477298637177050925,687971319,-1212461088,2022-05-02 17:47:47,4,86000000,0,0,1212,279,0,1,0
+5662566985137355690,7477298637177050925,48140695,-1178781667,2022-05-02 17:47:47,11,19000000,0,0,1301,291,0,1,0
+3646047056327432861,6658453249126145996,0,1931323897,2022-05-02 15:59:59,60,450000000,0,0,403,83,0,0,0
+3646047056327432861,6658453249126145996,1931323897,711219543,2022-05-02 15:59:59,57,453000000,0,0,649,153,0,0,0
+3646047056327432861,6658453249126145996,711219543,-1561583093,2022-05-02 15:59:59,4,466000000,0,0,643,153,0,0,0
+3646047056327432861,6658453249126145996,711219543,81837891,2022-05-02 15:59:59,24,486000000,0,0,643,153,0,0,0
+3646047056327432861,6658453249126145996,711219543,1963630877,2022-05-02 15:59:59,5,505000000,0,0,643,153,0,0,0
+3646047056327432861,6658453249126145996,711219543,-1370474143,2022-05-02 15:59:59,5,505000000,0,0,643,153,0,0,0
+3646047056327432861,6658453249126145996,711219543,1992379152,2022-05-02 15:59:59,17,473000000,0,500,644,153,0,0,0
+3646047056327432861,6658453249126145996,1992379152,360300743,2022-05-02 15:59:59,13,467000000,0,500,673,156,0,0,0

sample_dataset/train.csv

+traceIdHigh,traceIdLow,parentSpanId,spanId,startTime,duration,nanosecond,DBhash,status,operationName,serviceName
+7565692060307280094,4149727419449628353,0,-1160783149,2022-05-02 18:32:16,8,522000000,0,0,3,2
+7565692060307280094,4149727419449628353,-1160783149,-663518905,2022-05-02 18:32:16,5,525000000,0,0,6,2
+7565692060307280094,4149727419449628353,-663518905,292134812,2022-05-02 18:32:16,2,528000000,0,0,17,4
+7469867095312772506,9054005929149387783,0,-116188428,2022-05-02 15:59:34,9,401000000,0,0,3,2
+7469867095312772506,9054005929149387783,-116188428,-936201595,2022-05-02 15:59:34,6,404000000,0,0,6,2
+7469867095312772506,9054005929149387783,-936201595,203651095,2022-05-02 15:59:34,4,406000000,0,0,17,4
+1302365061014104190,4713847854262197922,0,147046579,2022-05-02 16:05:08,8,262000000,0,0,3,2
+1302365061014104190,4713847854262197922,147046579,-815705052,2022-05-02 16:05:08,5,265000000,0,0,6,2
+1302365061014104190,4713847854262197922,-815705052,-1767599182,2022-05-02 16:05:08,3,267000000,0,0,17,4
+4520607604792678538,90296779370241065,0,-1192584432,2022-05-02 15:03:15,10,800000000,0,0,3,2
+4520607604792678538,90296779370241065,-1192584432,2102356101,2022-05-02 15:03:15,7,803000000,0,0,6,2
+4520607604792678538,90296779370241065,2102356101,-1201056520,2022-05-02 15:03:15,5,805000000,0,0,17,4
+4301483373465897605,1921512395340371532,0,-116188428,2022-05-02 15:59:34,9,401000000,0,0,3,2
+4301483373465897605,1921512395340371532,-116188428,-936201595,2022-05-02 15:59:34,6,404000000,0,0,6,2
+4301483373465897605,1921512395340371532,-936201595,203651095,2022-05-02 15:59:34,4,406000000,0,0,17,4
+4516399473592649105,3122656494628515993,0,-940181392,2022-05-02 16:32:32,8,162000000,0,0,3,2
+4516399473592649105,3122656494628515993,-940181392,-1775474648,2022-05-02 16:32:32,6,164000000,0,0,6,2
+4516399473592649105,3122656494628515993,-1775474648,-614412147,2022-05-02 16:32:32,4,166000000,0,0,17,4
+2001615163061348505,1103669949841503114,0,-251538214,2022-05-02 17:19:09,7,523000000,0,0,3,2
+2001615163061348505,1103669949841503114,-251538214,-295534211,2022-05-02 17:19:09,5,525000000,0,0,6,2
+2001615163061348505,1103669949841503114,-295534211,-1868571256,2022-05-02 17:19:09,3,527000000,0,0,17,4

sample_dataset/val.csv

+traceIdHigh,traceIdLow,parentSpanId,spanId,startTime,duration,nanosecond,DBhash,status,operationName,serviceName
+3718860522599049613,8245757295869652015,0,744605799,2022-05-02 15:34:49,10,630000000,0,0,3,2
+3718860522599049613,8245757295869652015,744605799,258669622,2022-05-02 15:34:49,6,634000000,0,0,6,2
+3718860522599049613,8245757295869652015,258669622,-2140429749,2022-05-02 15:34:49,4,626000000,0,0,17,4
+171786705856344778,8432392905725602200,0,834436815,2022-05-02 15:38:25,22,928000000,0,0,3,2
+171786705856344778,8432392905725602200,834436815,1941867149,2022-05-02 15:38:25,19,931000000,0,0,6,2
+171786705856344778,8432392905725602200,1941867149,-2120879518,2022-05-02 15:38:25,17,933000000,0,0,17,4
+1884068908740226663,198674487451014796,0,-1192584432,2022-05-02 15:03:15,10,800000000,0,0,3,2
+1884068908740226663,198674487451014796,-1192584432,2102356101,2022-05-02 15:03:15,7,803000000,0,0,6,2
+1884068908740226663,198674487451014796,2102356101,-1201056520,2022-05-02 15:03:15,5,805000000,0,0,17,4
+6433112438253994909,8162558342722269388,0,-940181392,2022-05-02 16:32:32,8,162000000,0,0,3,2
+6433112438253994909,8162558342722269388,-940181392,-1775474648,2022-05-02 16:32:32,6,164000000,0,0,6,2
+6433112438253994909,8162558342722269388,-1775474648,-614412147,2022-05-02 16:32:32,4,166000000,0,0,17,4
+6877930537907535995,8837251782173988176,0,-112915630,2022-05-02 16:57:11,7,243000000,0,0,3,2
+6877930537907535995,8837251782173988176,-112915630,-1653931621,2022-05-02 16:57:11,4,236000000,0,0,6,2
+6877930537907535995,8837251782173988176,-1653931621,-48204267,2022-05-02 16:57:11,2,238000000,0,0,17,4
+790603823422444737,8049481303438475592,0,-269253468,2022-05-02 15:00:08,14,176000000,0,0,3,2
+790603823422444737,8049481303438475592,-269253468,1292285861,2022-05-02 15:00:08,11,179000000,0,0,6,2
+790603823422444737,8049481303438475592,1292285861,-205310853,2022-05-02 15:00:08,4,186000000,0,0,17,4
+4016062494274678058,802407659628105677,0,-1067224499,2022-05-02 15:14:32,20,520000000,0,0,3,2
+4016062494274678058,802407659628105677,-1067224499,1026915991,2022-05-02 15:14:32,17,523000000,0,0,6,2
+4016062494274678058,802407659628105677,1026915991,-1777842468,2022-05-02 15:14:32,15,525000000,0,0,17,4

test.sh

+echo "Usage: bash test.sh [model_path] [dataset_path]"
+echo "MODEL: $1"
+echo "DATASET: $2"
+python3 -m tracegnn.models.trace_vae.test evaluate-nll -M "$1" --use-train-val -D "$2" --device cpu --use-std-limit --std-limit-global

tracegnn/constants.py

+import os
+
+# if MIN_NODE_COUNT <= 2 <= MAX_NODE_COUNT, then the graph will be chosen
+MAX_NODE_COUNT = int(os.environ.get('MAX_NODE_COUNT', '32'))
+MAX_SPAN_COUNT = int(os.environ.get('MAX_SPAN_COUNT', '32'))
+
+# whether or not to use multi-dimensional latency codec?
+# If not set, will normalize the latency w.r.t. each operation.
+USE_MULTI_DIM_LATENCY_CODEC = os.environ.get('USE_MULTI_DIM_LATENCY_CODEC', '0') == '1'
+
+# If USE_MULTI_DIM_LATENCY_CODEC, then encode the codec parameters.
+MAX_LATENCY_DIM = int(os.environ.get('MAX_LATENCY_DIM', '5'))
+MAX_DEPTH = int(os.environ.get('MAX_DEPTH', '4'))

tracegnn/data/__init__.py

+from .bytes_db import *
+from .trace_graph import *
+from .trace_graph_db import *