diff --git a/.gitignore b/.gitignore index a2a7ce1..09d5e3e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +tmp logs target /.idea diff --git a/aws-certs.sh b/aws-certs.sh index 6adc130..b581383 100644 --- a/aws-certs.sh +++ b/aws-certs.sh @@ -2,6 +2,7 @@ # modified to import into cacerts so they're available without specifying a truststore # modified to not require perl +set -euxo pipefail mydir=tmp/certs if [ ! -e "${mydir}" ] @@ -12,13 +13,15 @@ fi storepassword=changeit curl -sS "https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem" > ${mydir}/global-bundle.pem -awk 'split_after == 1 {n++;split_after=0} /-----END CERTIFICATE-----/ {split_after=1}{print > "rds-ca-" n+1 ".pem"}' < ${mydir}/global-bundle.pem +#awk 'split_after == 1 {n++;split_after=0} /-----END CERTIFICATE-----/ {split_after=1}{print > "rds-ca-" n+1 ".pem"}' < ${mydir}/global-bundle.pem +cd ${mydir} && awk 'BEGIN {n=0; split_after=0} split_after == 1 {n++;split_after=0} /-----END CERTIFICATE-----/ {split_after=1}{print > ("rds-ca-" (n+1) ".pem")}' < global-bundle.pem && cd - -for CERT in rds-ca-*; do +#for CERT in rds-ca-*; do +for CERT in ${mydir}/rds-ca-*; do alias=$(openssl x509 -noout -subject -in $CERT | awk -F'CN=' '{print $2}') echo "Importing $alias" keytool -import -file ${CERT} -alias "${alias}" -storepass ${storepassword} -cacerts -noprompt rm $CERT done -rm ${mydir}/global-bundle.pem \ No newline at end of file +rm ${mydir}/global-bundle.pem diff --git a/db-scripts/00-init.sql b/db-scripts/00-init.sql new file mode 100644 index 0000000..b485c0c --- /dev/null +++ b/db-scripts/00-init.sql @@ -0,0 +1,30 @@ +CREATE TABLE account ( + id integer NOT NULL, + key character(32) NOT NULL, + email character varying(100) NOT NULL, + enabled boolean DEFAULT true, + staff boolean DEFAULT false, + created_at timestamp with time zone DEFAULT timezone('utc'::text, now()), + updated_at timestamp with time zone DEFAULT timezone('utc'::text, now()) +); + +CREATE SEQUENCE account_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE account_id_seq OWNED BY account.id; + +ALTER TABLE ONLY account ALTER COLUMN id SET DEFAULT nextval('account_id_seq'::regclass); + +ALTER TABLE ONLY account + ADD CONSTRAINT account_pkey PRIMARY KEY (id); + +CREATE INDEX account_email_idx ON account USING btree (email); + +CREATE UNIQUE INDEX account_key_idx ON account USING btree (key); + + + diff --git a/db-scripts/01-api-key.sql b/db-scripts/01-api-key.sql new file mode 100644 index 0000000..f7fc5c3 --- /dev/null +++ b/db-scripts/01-api-key.sql @@ -0,0 +1,8 @@ +insert into account (key, + email, + enabled, + staff) +values ('ool1Phovah1wie7juapheGoo9aNg1quu', + 'foo@bar.com', + true, + true); \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9734b5c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,63 @@ +services: + + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + volumes: + - ./es-scripts:/usr/share/elasticsearch/config/scripts + command: > + /bin/bash -c " + chmod +x /usr/share/elasticsearch/config/scripts/*.sh && + /usr/share/elasticsearch/config/scripts/init-indices.sh + " + depends_on: + - elasticsearch + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2 + volumes: + - ./tmp/elasticsearch:/usr/share/elasticsearch/data + environment: + #- network.host=localhost + - discovery.type=single-node + - bootstrap.system_call_filter=false + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + + ports: + - "9200:9200" + + # psql -h localhost -p 5432 -U postgres -f src/main/resources/00-init.sql + postgres: + image: postgres:16 + volumes: + - ./tmp/db:/var/lib/postgresql/data + - ./db-scripts:/docker-entrypoint-initdb.d + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=weebaeloo3taZiezaip6Uph4R + - POSTGRES_DB=postgres + ports: + - "5432:5432" + + api: + build: + context: . + dockerfile: Dockerfile + ports: + - "8080:8080" + environment: + - ELASTICSEARCH_URL=http://elasticsearch:9200 + - DATABASE_URL=jdbc:postgresql://postgres:5432/postgres + - ITEM_ELASTICSEARCH_URL=http://elasticsearch:9200/dpla_alias + - PSS_ELASTICSEARCH_URL=http://search.internal.dp.la:9200/dpla_pss + - EBOOK_ELASTICSEARCH_URL=http://elasticsearch:9200/dpla_ebooks + - POSTGRES_PASSWORD=weebaeloo3taZiezaip6Uph4R + - POSTGRES_SERVER=postgres + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - SSL_MODE=disable + - SSL=false + - SSL_FACTORY=org.postgresql.ssl.DefaultJavaSSLFactory + command: ["/usr/bin/java", "-jar", "/opt/api/dpla-api.jar"] + depends_on: + - elasticsearch + - postgres \ No newline at end of file diff --git a/es-scripts/init-indices.sh b/es-scripts/init-indices.sh new file mode 100755 index 0000000..ffc8c77 --- /dev/null +++ b/es-scripts/init-indices.sh @@ -0,0 +1,508 @@ +#!/bin/bash +until curl -s http://elasticsearch:9200 > /dev/null; do + sleep 1 +done + +response=$(curl -s -o /dev/null -w "%{http_code}" http://elasticsearch:9200/dpla_alias) + +if [ $response -eq 200 ]; then + exit 0 +fi + +curl -s -X PUT "http://elasticsearch:9200/dpla_alias?include_type_name=true" -H 'Content-Type: application/json' -d' +{ + "settings" : { + "index": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "max_result_window": 50000 + }, + "analysis": { + "normalizer": { + "canonicalsort": { + "type": "custom", + "char_filter": "canonicalsort_char_filter", + "filter": ["lowercase", "asciifolding"] + } + }, + "char_filter": { + "canonicalsort_char_filter": { + "type": "pattern_replace", + "pattern": "[^a-z0-9]+|(?:\\bthe\\b|\\bof\\b|\\band\\b|\\ba\\b|\\ban\\b)", + "replacement": "", + "flags": "CASE_INSENSITIVE" + }, + "editing_notation_filter": { + "type": "pattern_replace", + "pattern": "[\\[\\]]", + "replacement": "" + } + }, + "filter": { + "shingle": { + "max_shingle_size": "4", + "min_shingle_size": "2", + "output_unigrams": "true", + "type": "shingle" + } + }, + "analyzer": { + "mlt": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase", "stop", "snowball"] + }, + "suggestion": { + "type": "custom", + "tokenizer": "standard", + "char_filter": ["editing_notation_filter"], + "filter": ["lowercase", "asciifolding", "shingle"] + }, + "nicer_search": { + "type": "custom", + "tokenizer": "standard", + "char_filter": ["editing_notation_filter"], + "filter": ["lowercase", "asciifolding"] + } + } + } + }, + "mappings" : { + "item" : { + "dynamic" : "false", + "properties" : { + "@context" : { + "type" : "object", + "enabled" : false + }, + "@id" : { + "type" : "keyword" + }, + "admin" : { + "properties" : { + "contributingInstitution": { + "type": "keyword" + } + } + }, + "aggregatedCHO" : { + "type" : "object", + "enabled" : false + }, + "dataProvider" : { + "properties" : { + "@id" : { + "type" : "keyword" + }, + "name" : { + "type" : "text", + "copy_to": "admin.contributingInstitution", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + }, + "exactMatch": { + "type": "keyword" + } + } + }, + "hasView" : { + "properties" : { + "@id" : { + "type" : "keyword" + }, + "edmRights" : { + "type" : "text", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + }, + "format" : { + "type" : "keyword" + }, + "rights" : { + "type" : "keyword" + } + } + }, + "id" : { + "type" : "keyword" + }, + "iiifManifest" : { + "type" : "keyword" + }, + "ingestDate" : { + "type" : "object", + "enabled" : false + }, + "ingestType" : { + "type" : "object", + "enabled" : false + }, + "ingestionSequence" : { + "type" : "object", + "enabled" : false + }, + "intermediateProvider" : { + "type" : "text", + "copy_to": "admin.contributingInstitution", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + }, + "ipfs" : { + "type" : "keyword" + }, + "isPartOf" : { + "properties" : { + "@id" : { + "type" : "keyword" + }, + "name" : { + "type" : "text", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + } + } + }, + "isShownAt" : { + "type" : "keyword" + }, + "mediaMaster" : { + "type" : "keyword" + }, + "object" : { + "type" : "keyword" + }, + "originalRecord" : { + "type" : "object", + "enabled" : false + }, + "provider" : { + "properties" : { + "@id" : { + "type" : "keyword" + }, + "name" : { + "type" : "text", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + }, + "exactMatch": { + "type": "keyword" + } + } + }, + "rights" : { + "type" : "keyword" + }, + "rightsCategory" : { + "type" : "keyword" + }, + "tags" : { + "type" : "keyword" + }, + "sourceResource" : { + "properties" : { + "@id" : { + "type" : "object", + "enabled" : false + }, + "collection" : { + "properties" : { + "@id" : { + "type" : "keyword" + }, + "description" : { + "type" : "text" + }, + "id" : { + "type" : "keyword" + }, + "title" : { + "type" : "text", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + } + } + }, + "contributor" : { + "type" : "keyword" + }, + "creator" : { + "type" : "text" + }, + "date" : { + "properties" : { + "begin" : { + "type" : "date", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + }, + "ignore_malformed" : true, + "null_value" : "-9999", + "format" : "date_optional_time||yyyy||yyyy-MM||yyyy-MM-dd" + }, + "displayDate" : { + "type" : "object", + "enabled" : false + }, + "end" : { + "type" : "date", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + }, + "ignore_malformed" : true, + "null_value" : "-9999", + "format" : "date_optional_time||yyyy||yyyy-MM||yyyy-MM-dd" + } + } + }, + "description" : { + "type" : "text", + "analyzer": "nicer_search", + "fields" : { + "suggestion" : { + "type" : "text", + "analyzer" : "suggestion" + } + } + }, + "extent" : { + "type" : "keyword" + }, + "format" : { + "type" : "keyword" + }, + "genre" : { + "type" : "text" + }, + "identifier" : { + "type" : "text", + "index_options" : "docs" + }, + "isPartOf" : { + "type" : "object", + "enabled" : false + }, + "language" : { + "properties" : { + "iso639_3" : { + "type" : "keyword" + }, + "name" : { + "type" : "keyword", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + } + } + }, + "publisher" : { + "type" : "text", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + }, + "relation" : { + "type" : "text" + }, + "rights" : { + "type" : "text" + }, + "spatial" : { + "properties" : { + "city" : { + "type" : "text", + "analyzer": "nicer_search", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + }, + "coordinates" : { + "type" : "geo_point", + "ignore_malformed" : true + }, + "country" : { + "type" : "text", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + }, + "county" : { + "type" : "text", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + }, + "iso3166-2" : { + "type" : "keyword" + }, + "name" : { + "type" : "text", + "analyzer": "nicer_search", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + }, + "region" : { + "type" : "text", + "analyzer": "nicer_search", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + }, + "state" : { + "type" : "text", + "analyzer": "nicer_search", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + } + } + } + }, + "specType" : { + "type" : "keyword" + }, + "stateLocatedIn" : { + "type": "object", + "enabled": false + }, + "subject" : { + "properties" : { + "@id" : { + "type" : "keyword" + }, + "@type" : { + "type" : "keyword" + }, + "name" : { + "type" : "text", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + }, + "mlt": { + "type" : "text", + "analyzer" : "mlt" + }, + "suggestion" : { + "type" : "text", + "analyzer" : "suggestion" + } + } + }, + "scheme" : { + "type": "text", + "fields": { + "not_analyzed": { + "type": "keyword" + } + } + }, + "exactMatch" : { + "type": "keyword" + } + } + }, + "temporal" : { + "properties" : { + "begin" : { + "type" : "date", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + }, + "ignore_malformed" : true, + "null_value" : "-9999", + "format" : "date_optional_time||yyyy||yyyy-MM||yyyy-MM-dd" + }, + "displayDate" : { + "type" : "object", + "enabled" : false + }, + "encoding" : { + "type" : "object", + "enabled" : false + }, + "end" : { + "type" : "date", + "fields" : { + "not_analyzed" : { + "type" : "keyword" + } + }, + "ignore_malformed" : true, + "null_value" : "-9999", + "format" : "date_optional_time||yyyy||yyyy-MM||yyyy-MM-dd" + }, + "point" : { + "type" : "object", + "enabled" : false + } + } + }, + "title" : { + "type" : "text", + "analyzer": "nicer_search", + "fields": { + "not_analyzed": { + "type": "keyword", + "normalizer": "canonicalsort" + }, + "mlt": { + "type" : "text", + "analyzer" : "mlt" + }, + "suggestion" : { + "type" : "text", + "analyzer" : "suggestion" + } + } + }, + "type" : { + "type" : "keyword" + } + } + } + } + } + } +}'