Skip to content

Commit 681dfb3

Browse files
excelwangexcelwang
committed
update
1 parent c518a0b commit 681dfb3

15 files changed

+346
-274
lines changed

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
"# ambari-gstore"
1+
"# ambari-gstore"

‎configuration/ASERV.xml

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
<?xml version="1.0"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
4+
<configuration supports_adding_forbidden="true">
5+
<property>
6+
<name>download_url</name>
7+
<value>http://repo/hdp/aserv/</value>
8+
<description>the prefix of download url.</description>
9+
</property>
10+
<property>
11+
<name>config</name>
12+
<description>This is the jinja template for gwac/gwac_dbgen_cluster/config.sh</description>
13+
<value>
14+
#!/bin/bash
15+
#the linux user name
16+
username="root"
17+
#the host name of master node
18+
mastername="emr17fb191d"
19+
20+
templateTable="template_table" #gwac/ is root
21+
sourceFilePath="catalog.csv" #gwac/ is root
22+
abstarTemlatePath="abstar_template" #gwac/ is root
23+
24+
redisIP="emr17fb191d:7000"
25+
redisPath="/usr/bin"
26+
27+
detectorMaster="emr17fb191d:5555"
28+
detectorWorkDir=$HOME/gwac/detector
29+
30+
jobServerHost="emr17fb191d:8090"
31+
jobServerPath="/opt/spark-job-server"
32+
jobServerRunUser="hdfs"
33+
34+
queryEnginePath="../astroDB_cache/astro_engine-assembly-0.2.2.jar"
35+
36+
hbaseInsertPath=/data/gwac/astroBase
37+
38+
#sumpost is used for the communication between master and slaves
39+
sumpost=1984
40+
#we bulid a script stopGen.sh to stop the whole data generator.
41+
#stoppost is used for the communication between master and stopGen.sh.
42+
stoppost=1986
43+
#we bulid a script getSumData.sh to obtain the sum at the runtime,
44+
#including the total number of lines, the total data size of star tables, etc.
45+
#printpost is used for the communication between master and getSumData.sh
46+
printpost=1985
47+
#when each slave node start, we also start a daemon process to monitor and stop them.
48+
#stoppost is used for the communication between master and daemon processes of slaves.
49+
stoppostMC=1987
50+
</value>
51+
</property>
52+
<property>
53+
<name>nodehostname</name>
54+
<description>This is the jinja template for gwac/gwac_dbgen_cluster/nodehostname</description>
55+
<value>
56+
emre69bf7ed-208-10-0-0
57+
emrbf554c6f-208-26-0-4096
58+
</value>
59+
</property>
60+
</configuration>

‎configuration/astroserver.xml

-22
This file was deleted.

‎metainfo.xml

+41-70
Original file line numberDiff line numberDiff line change
@@ -3,91 +3,62 @@
33
<schemaVersion>2.0</schemaVersion>
44
<services>
55
<service>
6-
<name>ASTROSERVER</name>
7-
<displayName>Astroserver</displayName>
8-
<comment>Astroserver</comment>
9-
<version>1.0</version>
6+
<name>ASERV</name>
7+
<displayName>Aserv</displayName>
8+
<comment>a astro streaming data management system</comment>
9+
<version>0.0.1</version>
1010
<components>
1111
<component>
12-
<name>AJAGUAR_MASTER</name>
13-
<displayName>jaguar_master</displayName>
12+
<name>GWAC_DBGEN_CLUSTER_MASTER</name>
13+
<displayName>Gwac_dbgen_cluster Master</displayName>
1414
<category>MASTER</category>
15-
<cardinality>1</cardinality>
16-
<versionAdvertised>true</versionAdvertised>
17-
<timelineAppid>astroserver</timelineAppid>
15+
<cardinality>1</cardinality>
16+
<timelineAppid>aserv</timelineAppid>
1817
<commandScript>
19-
<script>scripts/jaguar.py</script>
18+
<script>scripts/gwac_dbgen_master.py</script>
2019
<scriptType>PYTHON</scriptType>
2120
<timeout>600</timeout>
2221
</commandScript>
2322
</component>
24-
<component>
25-
<name>BSQUIRREL_MASTER</name>
26-
<displayName>Squirrel_master</displayName>
27-
<category>MASTER</category>
28-
<cardinality>1</cardinality>
29-
<versionAdvertised>true</versionAdvertised>
30-
<timelineAppid>astroserver</timelineAppid>
31-
<commandScript>
32-
<script>scripts/squirrel.py</script>
33-
<scriptType>PYTHON</scriptType>
34-
<timeout>600</timeout>
35-
</commandScript>
36-
</component>
37-
<component>
38-
<name>CGWAC_DBGEN_MASTER</name>
39-
<displayName>gwac_dbgen_master</displayName>
40-
<category>MASTER</category>
41-
<cardinality>1</cardinality>
42-
<versionAdvertised>true</versionAdvertised>
43-
<timelineAppid>astroserver</timelineAppid>
44-
<commandScript>
45-
<script>scripts/gwac_dbgen.py</script>
46-
<scriptType>PYTHON</scriptType>
47-
<timeout>600</timeout>
48-
</commandScript>
49-
</component>
50-
<component>
51-
<name>AJAGUAR_SALVE</name>
52-
<displayName>jaguar_slave</displayName>
53-
<category>SLAVE</category>
54-
<cardinality>1+</cardinality>
55-
<versionAdvertised>true</versionAdvertised>
56-
<timelineAppid>astroserver</timelineAppid>
57-
<commandScript>
58-
<script>scripts/jaguar.py</script>
59-
<scriptType>PYTHON</scriptType>
60-
<timeout>600</timeout>
61-
</commandScript>
62-
</component>
63-
<component>
64-
<name>BSQUIRREL_SLAVE</name>
65-
<displayName>Squirrel_slave</displayName>
66-
<category>SLAVE</category>
67-
<cardinality>1+</cardinality>
68-
<versionAdvertised>true</versionAdvertised>
69-
<timelineAppid>astroserver</timelineAppid>
70-
<commandScript>
71-
<script>scripts/squirrel.py</script>
72-
<scriptType>PYTHON</scriptType>
73-
<timeout>600</timeout>
74-
</commandScript>
75-
</component>
76-
<component>
77-
<name>CGWAC_DBGEN_SALVE</name>
78-
<displayName>gwac_dbgen_slave</displayName>
79-
<category>SLAVE</category>
80-
<cardinality>1+</cardinality>
81-
<versionAdvertised>true</versionAdvertised>
82-
<timelineAppid>astroserver</timelineAppid>
23+
<component>
24+
<name>GWAC_DBGEN_CLUSTER_SLAVE</name>
25+
<displayName>Gwac_dbgen_cluster Slave</displayName>
26+
<category>CLIENT</category>
27+
<cardinality>1+</cardinality>
8328
<commandScript>
84-
<script>scripts/gwac_dbgen.py</script>
29+
<script>scripts/gwac_dbgen_slave.py</script>
8530
<scriptType>PYTHON</scriptType>
8631
<timeout>600</timeout>
8732
</commandScript>
8833
</component>
8934
</components>
9035

36+
<requiredServices>
37+
<service>SPARK</service>
38+
<service>SPARKJS</service>
39+
<service>HBASE</service>
40+
<service>REDIS</service>
41+
</requiredServices>
42+
43+
<osSpecifics>
44+
<osSpecific>
45+
<osFamily>redhat5,redhat6,redhat7</osFamily>
46+
<packages>
47+
<package>
48+
<name>gcc-c++</name>
49+
</package>
50+
<package>
51+
<name>libtool</name>
52+
</package>
53+
<package>
54+
<name>epel-release</name>
55+
</package>
56+
<package>
57+
<name>python2-pip</name>
58+
</package>
59+
</packages>
60+
</osSpecific>
61+
</osSpecifics>
9162
<quickLinksConfigurations>
9263
<quickLinksConfiguration>
9364
<fileName>quicklinks.json</fileName>

‎metrics.json

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"GWAC_DBGEN_CLUSTER_MASTER": {
3+
"Component": [
4+
{
5+
"type": "ganglia",
6+
"metrics": {
7+
"default": {
8+
"metrics/aserv/line": {
9+
"metric": "line",
10+
"pointInTime": true,
11+
"temporal": true
12+
},
13+
"metrics/aserv/size": {
14+
"metric": "size",
15+
"pointInTime": true,
16+
"temporal": true
17+
}
18+
}
19+
}
20+
}
21+
]
22+
}
23+
}

‎package/scripts/aserv_metric_send.sh

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/sh
2+
url=http://$1:6188/ws/v1/timeline/metrics
3+
hostname=$2
4+
appid='aserv'
5+
cd ~/gwac/gwac_dbgen_cluster
6+
while [ 1 ]
7+
do
8+
./getSumData.sh
9+
arr=(`cat stalog.txt`)
10+
line=${arr[0]}
11+
size=${arr[1]}
12+
latesttime="${arr[2]} ${arr[3]}"
13+
14+
millon_time=$(( $(date +%s%N) / 1000000 ))
15+
json="{
16+
\"metrics\": [
17+
{
18+
\"metricname\": \"line\",
19+
\"appid\": \"${appid}\",
20+
\"hostname\": \"${hostname}\",
21+
\"timestamp\": ${millon_time},
22+
\"starttime\": ${millon_time},
23+
\"metrics\": {
24+
\"${millon_time}\": ${line}
25+
}
26+
},
27+
{
28+
\"metricname\": \"size\",
29+
\"appid\": \"${appid}\",
30+
\"hostname\": \"${hostname}\",
31+
\"timestamp\": ${millon_time},
32+
\"starttime\": ${millon_time},
33+
\"metrics\": {
34+
\"${millon_time}\": ${size}
35+
}
36+
}
37+
]
38+
}"
39+
40+
# echo $json
41+
#|tee -a /root/my_metric.log
42+
curl -i -X POST -H "Content-Type: application/json" -d "${json}" ${url}
43+
sleep 5
44+
done

‎package/scripts/gwac_dbgen.py

-42
This file was deleted.

‎package/scripts/gwac_dbgen_master.py

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# -*- coding: utf-8 -*-
2+
import os
3+
import base64
4+
from time import sleep
5+
from resource_management import *
6+
7+
class DBGENMaster(Script):
8+
9+
def install(self, env):
10+
import params
11+
env.set_params(params)
12+
print 'installed'
13+
Directory(["/data/"],
14+
mode=0755,
15+
cd_access='a',
16+
create_parents=True
17+
)
18+
Execute("pip install pyfits==3.3 numpy astropy==2.0.7 scipy", ignore_failures=False)
19+
Execute("easy_install "+params.download_url+"/pywcs-1.12-py2.7-linux-x86_64.egg", ignore_failures=False)
20+
Execute("cd /data/;wget "+params.download_url+"/gwac.1807130845.tar.gz;tar -xzf gwac*.gz;rm -rf gwac.1807130845.tar.gz;ln -sf /data/gwac ~/gwac", ignore_failures=False)
21+
Execute("cd /tmp/;wget "+params.download_url+"/go1.10.3.linux-amd64.tar.gz;tar -C /usr/local -xzf /tmp/go1.10.3.linux-amd64.tar.gz;echo 'export PATH=$PATH:/usr/local/go/bin'>>~/.bashrc", ignore_failures=False)
22+
Execute("cd /tmp/;wget "+params.download_url+"/libsodium-1.0.13.tar.gz;tar -xzvf libsodium-1.0.13.tar.gz;cd libsodium-1.0.13;make && make install", ignore_failures=False)
23+
Execute("cd /tmp/;wget "+params.download_url+"/libzmq.e6f908.tar.gz;tar -xzvf libzmq.e6f908.tar.gz;cd libzmq;make && make install;ldconfig", ignore_failures=False)
24+
Execute("cd /tmp/;wget "+params.download_url+"/czmq.e6f908.tar.gz;tar -xzvf czmq.e6f908.tar.gz;cd czmq;make && make install;ldconfig", ignore_failures=False)
25+
Execute("echo export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib>>~/.bashrc", ignore_failures=False)
26+
27+
def configure(self, env):
28+
import params
29+
env.set_params(params)
30+
server_cnf_config = InlineTemplate(params.server_cnf_config)
31+
server_cnf_nodehostname = InlineTemplate(params.server_cnf_nodehostname)
32+
File("/data/gwac/gwac_dbgen_cluster/config.sh", content=server_cnf_config)
33+
File("/data/gwac/gwac_dbgen_cluster/nodehostname", content=server_cnf_nodehostname)
34+
35+
def start(self, env):
36+
import params
37+
env.set_params(params)
38+
self.configure(env)
39+
service_packagedir = params.service_packagedir
40+
Execute('find '+params.service_packagedir+' -iname "*.sh" | xargs chmod +x')
41+
Execute(format("echo \"nohup {service_packagedir}/scripts/aserv_metric_send.sh {collector_host} {current_host_name} &\"|at now +1 min"))
42+
Execute("echo \"cd ~/gwac/gwac_dbgen_cluster/ && nohup ./sumLineAndDataSize.sh 10000 2>&1 >/dev/null &\"|at now +1 min");sleep(63)
43+
Execute("rm -rf /tmp/gwac_dbgen_cluster_master.pid;ps -ef | grep -v grep | grep sumLineAndDataSize.sh | awk '{print $2}'> /tmp/gwac_dbgen_cluster_master.pid")
44+
45+
def stop(self, env):
46+
Execute("cd ~/gwac/gwac_dbgen_cluster/ && ./stopGen.sh force", ignore_failures=True)
47+
sleep(10)
48+
Execute("ps -ef|stopGen.sh |grep -v grep|cut -c 9-15|xargs kill -9 ", ignore_failures=True)
49+
cmd = format("ps -ef|grep aserv_metric_send.sh |grep -v grep|cut -c 9-15|xargs kill -9 ")
50+
Execute(cmd, ignore_failures=True)
51+
cmd = format("ps -ef|detectorMaster |grep -v grep|cut -c 9-15|xargs kill -9 ")
52+
Execute(cmd, ignore_failures=True)
53+
54+
def restart(self, env):
55+
self.stop(env)
56+
self.start(env)
57+
58+
def status(self, env):
59+
check_process_status("/tmp/gwac_dbgen_cluster_master.pid")
60+
61+
62+
if __name__ == "__main__":
63+
DBGENMaster().execute()

0 commit comments

Comments
 (0)