Skip to content

Commit 3988a3b

Browse files
authored
Set up production necessities for deploying Safegraph (#259)
* Add updated encrypted credentials * Add wip_signal to production params template * Add newline * Run on 12 cores * Add shell script to run the indicator
1 parent fbe27a8 commit 3988a3b

File tree

3 files changed

+46
-36
lines changed

3 files changed

+46
-36
lines changed

ansible/templates/safegraph-params-prod.json.j2

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,10 @@
33
"raw_data_dir": "/common/safegraph",
44
"export_dir": "./receiving",
55
"cache_dir": "./cache",
6-
"n_core": "6",
6+
"n_core": "12",
77
"aws_access_key_id": "{{ safegraph_aws_access_key_id }}",
88
"aws_secret_access_key": "{{ safegraph_aws_secret_access_key }}",
99
"aws_default_region": "us-east-1",
10-
"aws_endpoint": "https://s3.wasabisys.com"
11-
}
10+
"aws_endpoint": "https://s3.wasabisys.com",
11+
"wip_signal": ""
12+
}

ansible/vault.yaml

Lines changed: 21 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,22 @@
11
$ANSIBLE_VAULT;1.1;AES256
2-
<<<<<<< HEAD
3-
30613131613362333037373437316261643833316430643638623461613137346536616262313232
4-
3666623130633831343831353265633338336363376664340a646639323034653632333866616561
5-
35656432653066633064386563356361313737333563376231623134623462663033383636346537
6-
6139373862666635370a393230316563366230326130323135366335376634373364623831323638
7-
34316133303664373135633331383530353362323263313932346232666164333033666564313061
8-
33346637363561633234356137633364633661356433643839363363383662396437363761653261
9-
35366164376466373631363532313337666230656162316561343263336335633164353565333565
10-
36613539666536643939396664653437343337626332353366303366313665616361383639633030
11-
64643834613562666134626262356330313134363239363762373362373462333435366331393466
12-
37336134306135363832626434366434376165343932316139373765346138616535313566373832
13-
63346530363266306333633535653233623436653031383536363133613165303566393133316434
14-
36343765326562383364396638643436303762666166663235646166386165326163616439313933
15-
63356565373432653165336562653833626536363261313138313532383030306363626465643662
16-
66333439653630346534663138323437306466653164336135343338393164653334626535303163
17-
623765666431333132653565303231373461
18-
=======
19-
66386163643862646634343162646465663762643034303563333833633661333932646164656462
20-
6166646131623132393238336263623562373065643633310a663232373237396361623462613333
21-
62373663383565623263306539636431623230633065626363666531366662363065343066363031
22-
3738616663336665340a326138333634306137363837396366303861663064326333613662656630
23-
62306331646637326637363766366237663037306665343761643263646663316535343561623137
24-
63313365653535393639626465343232396261643239303430383138633135346466323834336665
25-
33633064353034613836313265613466623961373565363835343430373138376336363966316365
26-
35663664396436313432376264316663326130306134326231303234393561643436623039613136
27-
63366638396262383762383336643930343661636461646162653734336334306239383132643435
28-
39333665643738643966356431333830646561353263353063326330643731616130396466343339
29-
39346437653063303336626663623835613938633834396430353634383366386237353862643766
30-
37393738353231666565303031393839306463373461393761653866653330646534393832303264
31-
30323038646166366465396235623731343539313633326539663966333437623733626131653437
32-
62326632656462383835656235373664366566343866383938343639613737623631616231616135
33-
633863383761366461363532353137323936
34-
>>>>>>> main
2+
39633436633363346633333638666438316131313337333132396634393538383432623239316463
3+
3435333238376331383439366161656639353039326163370a376363633535623363383233646533
4+
63363865646561323132663032383331346332373364333465643330616638623466333039623831
5+
6530663236313234360a336264656239383166663934303335386238386139386132626165386138
6+
32663164326237323534636263663263666634383339613362633939323565356437663666653436
7+
31353362316334313561333430626361616337643133346664636434313664373333653839323630
8+
65346331383135656135386263643564333063626563336365333865663333353337393866666139
9+
64613735663363323938633161666662653161633835383832656164343836383339376661396332
10+
66353131373265373931366130383632633466363036373562363232663162333966316563373535
11+
65343336363732303132366335616335333334373063313562336330336661353239646533356461
12+
62313365633336613037626261353639323937363066363062356234653631346233373965636461
13+
63326237663537363338346566326232353632663463386135393535343436373335393430393865
14+
33393631623762636230656263363462346561323064653561393666373735313836666238323238
15+
66366564666266343636663666386566336637373036633966643961346636373066356632326464
16+
63336565656666336436383938346461646431353265353133633736363761623634346262616436
17+
61653633326333356330626638386665313865343233393637623662383634346534326537623662
18+
34326633623431343835346339656335386330333664373166313766366339663736376261343965
19+
63616461666230616131326537373130313239663931313330356538356161333537666237376362
20+
64613232333834303737323438616437303666643166383439393030316533343530363863613034
21+
39653761626439356133393164363561316535633230633438316137623333376633663665393634
22+
63333161376263613766353030616336386531303565346263366239653232333764

safegraph/run-safegraph.sh

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Run the Safegraph indicator
4+
#
5+
6+
set -eo pipefail
7+
8+
# Purge the receiving directory.
9+
echo "Purging ./receiving..."
10+
rm -f ./receiving/*
11+
12+
# Run the indicator code.
13+
echo "Running the indicator..."
14+
env/bin/python -m delphi_safegraph
15+
16+
# Copy the files to the ingestion directory.
17+
#scp $(date +"receiving/%Y%m*") delphi.midas.cs.cmu.edu:/common/covidcast/receiving/safegraph/
18+
#scp $(date --date='-1 month' +"receiving/%Y%m*") delphi.midas.cs.cmu.edu:/common/covidcast/receiving/safegraph/
19+
echo "Copying files to the ingestion directory..."
20+
cp $(date +"receiving/%Y%m*") ./test-output/safegraph 2>/dev/null # Hack to make cp care less about missing files.
21+
cp $(date --date='-1 month' +"receiving/%Y%m*") ./test-output/safegraph

0 commit comments

Comments
 (0)