Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
53 commits
Select commit Hold shift + click to select a range
da6e22e
Clean billing calculator code initial commit
mapsacosta Mar 9, 2021
5d383c0
adding launcher and cvmfeexec wrapper for Stampede2
hufnagel Jul 12, 2021
3203fcd
Merge pull request #1 from hufnagel/stampede2
mapsacosta Jul 12, 2021
8486bba
add Frontera launcher and cvmfs wrapper
hufnagel Jul 15, 2021
62020dc
Merge pull request #2 from hufnagel/Frontera
mapsacosta Jul 19, 2021
6dcae48
Remove billing-calculator in favor of https://github.com/HEPCloud/bil…
mapsacosta Jul 19, 2021
710830e
adding Theta specific files
hufnagel Jul 31, 2021
43629fc
added site_specific to the list of projects
mambelli Sep 23, 2021
f962a70
Merge pull request #3 from hufnagel/Theta
mambelli Oct 29, 2021
0b6790a
Add README to de_monitoring
shreyb Feb 9, 2022
f102ee0
Added filebeat config
shreyb Feb 10, 2022
2b981f7
Added logstash digest config
shreyb Feb 10, 2022
654127d
Added elasticsearch storage logstash (store/)
shreyb Feb 10, 2022
96b0703
Moved filebeat config into its own directory
shreyb Feb 10, 2022
53dca82
Merge pull request #4 from shreyb/master
mambelli Feb 10, 2022
aea1f37
Add template configuration files for integration test
vitodb Feb 17, 2022
e9d76fc
Merge pull request #5 from vitodb/master
vitodb Feb 17, 2022
9395c19
Reorganized configuration templates and added condor_mapfile
vitodb Feb 17, 2022
93dd698
Merge pull request #6 from vitodb/master
vitodb Feb 17, 2022
6e36ffa
Added instructions for basic setup of filebeat
shreyb Feb 17, 2022
8126e7b
Merge pull request #7 from shreyb/master
shreyb Feb 17, 2022
4fbdd74
Added steps for basic setup of digest topic logstash
shreyb Feb 21, 2022
7e57dc7
Merge branch 'HEPCloud:master' into master
shreyb Feb 21, 2022
b05f660
Added steps for basic setup of digest topic logstash
shreyb Feb 21, 2022
84ca4ef
Added steps for basic setup of store logstash
shreyb Feb 24, 2022
107f593
Merge branch 'master_copy'
shreyb Feb 24, 2022
62825c1
Merge pull request #9 from shreyb/master
shreyb Feb 24, 2022
4508dc9
Update README.md
vitodb Feb 24, 2022
e52b10d
update DNs
vitodb May 31, 2022
f7dccb6
Merge pull request #10 from vitodb/master
vitodb May 31, 2022
fd69052
update proxy path
vitodb Jun 1, 2022
21030aa
Merge pull request #11 from vitodb/master
vitodb Jun 1, 2022
7a0e7c7
Adding folder with DE configuration templates
vitodb Jul 21, 2022
3aac1ab
test Shrijan
shrijan-swaminathan Aug 5, 2022
85acd04
swamina7: cloud benchmarking tarball with scripts
Aug 5, 2022
4ad6789
Merge branch 'cloudBenchmarking' of https://github.com/HEPCloud/contr…
Aug 5, 2022
b5f43a3
swamina7: benchmarking script creation
Aug 5, 2022
4c3bb58
swamina7: HS06 results for AWS and Google
Aug 5, 2022
98cf5ee
swamina7: spot price changed files
Aug 12, 2022
8fce58d
Merge pull request #12 from vitodb/config_template
mambelli Aug 18, 2022
175ce27
Update DE configs for master branch
vitodb Sep 22, 2022
540eb00
Merge pull request #13 from vitodb/vito_config_updates
vitodb Sep 22, 2022
0389b0d
Create test.txt
shrijan-swaminathan Jan 3, 2023
567984a
Add files via upload
shrijan-swaminathan Jan 3, 2023
e0449ea
Add files via upload
shrijan-swaminathan Jan 3, 2023
a3bfc7a
Delete test.txt
shrijan-swaminathan Jan 3, 2023
6ae029e
Move existing DE configuration files to EL7 folder
Jun 9, 2023
3cd1769
Add DE configuration templates for EL9
Jun 9, 2023
75004d7
Used different names for the TEMPLATE elements and added explanation
mambelli Jul 4, 2023
355f7b7
Fixed Typo
skylerfoster67 Jul 5, 2023
5cf5bbc
Added sample prometheus configuration for decisionengine metrics
shreyb Aug 31, 2023
1007252
Corrected default metrics path
shreyb Aug 31, 2023
fe76a11
Merge branch 'master' into master
shreyb Nov 9, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cloud_benchmarking/script_to_follow_GCloud
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# ON YOUR MACHINE (ON FERMICLOUD 241):
# ON YOUR MACHINE (ON ***REMOVED***):
echo "Enter instance name: (E.g: n1_test)"
read inst_name

Expand All @@ -14,7 +14,7 @@ gcloud compute scp oldbenchmark.tar run_DB12.sh ~/GoogleCloudBenchmarking/full_b
bash full_benchmark_for_GCloud.sh

######################################################################################################################
# ON YOUR MACHINE (FERMICLOUD 241):
# ON YOUR MACHINE (***REMOVED***):
#
# save results in a corresponding folder on this machine
# ONLY DO THIS AFTER A FEW HOURS!!!!
Expand Down
4 changes: 2 additions & 2 deletions de_monitoring/digest/logstash.conf.digest
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# This conf file is to take raw dCache billing data from Kafka, clean it up, and write it to a digest topic in Kafka
input {
kafka {
bootstrap_servers => "lssrv02.fnal.gov:9092"
bootstrap_servers => ***REMOVED***
topics => ["test.hepcloud.de"]
auto_offset_reset => "earliest"
decorate_events => true
Expand All @@ -24,7 +24,7 @@ filter {
output {
# stdout {codec=>"rubydebug"}
kafka {
bootstrap_servers => "lssrv02.fnal.gov:9092"
bootstrap_servers => ***REMOVED***
topic_id => "digest.test.hepcloud.de"
codec => "json"
}
Expand Down
2 changes: 1 addition & 1 deletion de_monitoring/filebeat/filebeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ setup.kibana:
#----------------------------- Kafka output --------------------------------
output.kafka:
# initial brokers for reading cluster metadata
hosts: ["lssrv02.fnal.gov:9092", ]
hosts: [***REMOVED***, ]

# message topic selection + partitioning
# topic: '%{[fields.log_topic]}'
Expand Down
2 changes: 1 addition & 1 deletion de_monitoring/store/logstash.conf
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
input {
kafka {
bootstrap_servers => "lssrv02:9092"
bootstrap_servers => ***REMOVED***
topics => [ "test.hepcloud.de" ]
auto_offset_reset => "earliest" # After testing, change to latest
decorate_events => true
Expand Down
2 changes: 1 addition & 1 deletion integration_test/condor/condor_mapfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
GSI "^/DC=org/DC=incommon/C=US/ST=Illinois/O=Fermi Research Alliance/OU=Fermilab/CN=@FERMICLOUDNODE@\.fnal\.gov$" decisionengine_service
GSI "^/DC=org/DC=cilogon/C=US/O=Fermi National Accelerator Laboratory/OU=People/CN=@First Last Name@/CN=UID:@username@" decisionengine_service
GSI "^/DC=org/DC=incommon/C=US/ST=Illinois/O=Fermi Research Alliance/OU=Fermilab/CN=fermicloud576\.fnal\.gov$" gfactory
GSI "^/DC=org/DC=incommon/C=US/ST=Illinois/O=Fermi Research Alliance/OU=Fermilab/CN=***REMOVED***$" gfactory
GSI (.*) anonymous
FS (.*) \1
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
module: "decisionengine_modules.htcondor.sources.job_q",
parameters: {
condor_config: "/etc/condor/condor_config",
collector_host: "@FERMICLOUDNODE@.fnal.gov",
collector_host: ***REMOVED***,
schedds: [
"@FERMICLOUDNODE@.fnal.gov"
***REMOVED***
],
constraint: "True",
classad_attrs: [
Expand Down Expand Up @@ -163,7 +163,7 @@
"TotalSlotCpus":0,
"GLIDEIN_CredentialIdentifier":''
},
collector_host: "fermicloud576.fnal.gov",
collector_host: ***REMOVED***,
condor_config: "/etc/condor/condor_config"
},
max_attempts: 100,
Expand Down Expand Up @@ -194,7 +194,7 @@
name: "JobClusteringPublisher",
parameters: {
publish_to_graphite: true,
graphite_host: "fifemondata.fnal.gov",
graphite_host: ***REMOVED***,
graphite_port: 2004,
graphite_context: "hepcloud.de.@FERMICLOUDNODE@.glideinwms",
output_file: "/etc/decisionengine/modules.data/job_cluster_totals.csv",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ local channels = [
condor_config: "/etc/condor/condor_config",
factories: [
{
collector_host: "fermicloud576.fnal.gov",
collector_host: ***REMOVED***,
classad_attrs: []
},
],
Expand All @@ -24,7 +24,7 @@ local channels = [
condor_config: "/etc/condor/condor_config",
factories: [
{
collector_host: "fermicloud576.fnal.gov",
collector_host: ***REMOVED***,
classad_attrs: [],
correction_map: {
"GLIDEIN_Resource_Slots":'',
Expand Down
18 changes: 9 additions & 9 deletions integration_test/decisionengine/glideinwms.libsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,11 @@
"match_attrs": {},
"collectors": [
{
"DN": "/DC=org/DC=incommon/C=US/ST=Illinois/O=Fermi Research Alliance/OU=Fermilab/CN=fermicloud576.fnal.gov",
"DN": ***REMOVED***,
"comment": "Test Factory",
"factory_identity": "gfactory@fermicloud576.fnal.gov",
"my_identity": "decisionengine_service@fermicloud576.fnal.gov",
"node": "fermicloud576.fnal.gov"
"factory_identity": ***REMOVED***,
"my_identity": ***REMOVED***,
"node": ***REMOVED***
}
]
},
Expand All @@ -82,8 +82,8 @@
"match_attrs": {},
"schedds": [
{
"DN": "/DC=org/DC=incommon/C=US/ST=Illinois/O=Fermi Research Alliance/OU=Fermilab/CN=@FERMICLOUDNODE@.fnal.gov",
"fullname": "@FERMICLOUDNODE@.fnal.gov"
"DN": ***REMOVED***,
"fullname": ***REMOVED***
}
]
}
Expand All @@ -103,8 +103,8 @@

"security": {
"classad_proxy": "/var/de/fe_proxy",
"comment": "Test DE at @FERMICLOUDNODE@.fnal.gov",
"proxy_DN": "/DC=org/DC=incommon/C=US/ST=Illinois/O=Fermi Research Alliance/OU=Fermilab/CN=@FERMICLOUDNODE@.fnal.gov",
"comment": ***REMOVED***,
"proxy_DN": ***REMOVED***,
"proxy_selection_plugin": "ProxyAll",
"security_name": "decisionengine_service",
"sym_key": "aes_256_cbc",
Expand All @@ -121,7 +121,7 @@
"stage": {
"base_dir": "/var/lib/gwms-frontend/web-area/stage",
"use_symlink": "True",
"web_base_url": "http://@FERMICLOUDNODE@.fnal.gov/vofrontend/stage"
"web_base_url": ***REMOVED***
},

"work": {
Expand Down
2 changes: 1 addition & 1 deletion site_specific/Theta/customize.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
#

awk --file `dirname $0`/customhelps.awk --source '{
setoption("cache_peer", "theta-proxy.tmi.alcf.anl.gov parent 3128 0 no-query")
setoption("cache_peer", "***REMOVED*** parent 3128 0 no-query")
setoption("acl NET_LOCAL src", "127.0.0.1/32")
setoption("cache_mem", "128 MB")
setoptionparameter("cache_dir", 3, "10000")
Expand Down
2 changes: 1 addition & 1 deletion site_specific/Theta/default.local
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
CVMFS_HTTP_PROXY="http://theta-proxy.tmi.alcf.anl.gov:3128"
CVMFS_HTTP_PROXY="http://***REMOVED***:3128"
CVMFS_CACHE_BASE=/local/scratch/uscms/cvmfs-cache
CVMFS_QUOTA_LIMIT=10000
CMS_LOCAL_SITE=T3_US_ANL