feat: add interlinear PDF export pipeline

Dockerfile.job-worker

@@ -6,14 +6,21 @@ COPY package*.json .
 COPY patches/ ./patches/
 RUN npm ci
 COPY . .
-RUN npm run build:job-worker
+RUN npm run build:job-worker && mkdir -p /app/dist/fonts && cp src/fonts/*.ttf /app/dist/fonts/
 
 FROM public.ecr.aws/lambda/nodejs:18 AS runner
 ENV NODE_ENV=production
+ENV PDFKIT_DATA_DIR=/var/task/data
 COPY --from=build /app/dist/job-worker.js ${LAMBDA_TASK_ROOT}
+COPY --from=build /app/node_modules/pdfkit/js/data /var/task/data
+COPY --from=build /app/dist/fonts /var/task/fonts
+COPY --from=build /app/src/fonts /var/task/fonts
 CMD ["job-worker.handler"]
 
 FROM public.ecr.aws/lambda/nodejs:18 AS dev
+ENV PDFKIT_DATA_DIR=/var/task/data
 COPY --from=build /app/dist/job-worker.js ${LAMBDA_TASK_ROOT}
+COPY --from=build /app/node_modules/pdfkit/js/data /var/task/data
+COPY --from=build /app/dist/fonts /var/task/fonts
+COPY --from=build /app/src/fonts /var/task/fonts
 CMD ["job-worker.handler"]
-

compose.local.yaml

@@ -0,0 +1,57 @@
+services:
+  server:
+    environment:
+      AWS_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: test
+      AWS_ENDPOINT_URL_S3: http://localstack:4566
+      AWS_S3_FORCE_PATH_STYLE: "true"
+    depends_on:
+      localstack:
+        condition: service_healthy
+
+  job_worker:
+    environment:
+      AWS_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: test
+      AWS_ENDPOINT_URL_S3: http://localstack:4566
+      AWS_S3_FORCE_PATH_STYLE: "true"
+      EXPORT_PUBLIC_S3_ENDPOINT: http://localhost:4566
+    depends_on:
+      localstack:
+        condition: service_healthy
+
+  localstack:
+    image: localstack/localstack:latest
+    environment:
+      - SERVICES=s3
+      - DEFAULT_REGION=us-east-1
+      - LS_LOG=warn
+    ports:
+      - 4566:4566
+    healthcheck:
+      test: ["CMD", "bash", "-c", "awslocal s3 ls >/dev/null 2>&1"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  localstack-init:
+    image: amazon/aws-cli:2.32.11
+    depends_on:
+      localstack:
+        condition: service_healthy
+    environment:
+      AWS_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: test
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - >
+        until aws --endpoint-url http://localstack:4566 s3api list-buckets >/dev/null 2>&1; do
+          echo "Waiting for localstack S3...";
+          sleep 2;
+        done;
+        aws --endpoint-url http://localstack:4566 s3 mb s3://gbt-exports-local >/dev/null 2>&1 || true;
+        echo "Localstack S3 bucket ready";
+    restart: "no"

db/migrations/25-12-06-add-export-request.sql

@@ -0,0 +1,36 @@
+create type export_request_status as enum ('PENDING', 'IN_PROGRESS', 'COMPLETE', 'FAILED');
+
+create table export_request (
+  id uuid primary key default generate_ulid(),
+  language_id uuid not null references language(id) on delete cascade,
+  book_id integer references book(id),
+  chapters integer[],
+  layout text not null default 'standard',
+  status export_request_status not null default 'PENDING',
+  job_id uuid references job(id),
+  download_url text,
+  expires_at timestamptz,
+  requested_by uuid not null references users(id) on delete cascade,
+  requested_at timestamptz not null default now(),
+  completed_at timestamptz,
+  export_key text
+);
+
+create index export_request_status_idx on export_request(status);
+create index export_request_requested_by_idx on export_request(requested_by);
+create index export_request_expires_at_idx on export_request(expires_at);
+
+create table if not exists export_request_book (
+  request_id uuid not null references export_request(id) on delete cascade,
+  book_id integer not null references book(id),
+  chapters integer[] not null,
+  primary key (request_id, book_id)
+);
+
+create index if not exists export_request_book_request_idx on export_request_book(request_id);
+
+insert into job_type (name)
+values
+  ('export_interlinear_pdf'),
+  ('cleanup_exports')
+on conflict (name) do nothing;

db/migrations/25-12-14-snapshot-interlinear-pdf-job-type.sql

@@ -0,0 +1,8 @@
+select setval(
+    pg_get_serial_sequence('job_type', 'id'),
+    (select coalesce(max(id), 0) from job_type)
+);
+
+insert into job_type (name)
+values ('create_snapshot_interlinear_pdf')
+on conflict (name) do nothing;

db/scripts/schema.sql

@@ -114,6 +114,17 @@ CREATE TYPE public.text_direction AS ENUM (
     'rtl'
 );
 
+--
+-- Name: export_request_status; Type: TYPE; Schema: public; Owner: -
+--
+
+CREATE TYPE public.export_request_status AS ENUM (
+    'PENDING',
+    'IN_PROGRESS',
+    'COMPLETE',
+    'FAILED'
+);
+
 
 --
 -- Name: user_status; Type: TYPE; Schema: public; Owner: -
@@ -577,6 +588,37 @@ CREATE TABLE public.language_member_role (
     role public.language_role NOT NULL
 );
 
+--
+-- Name: export_request; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.export_request (
+    id uuid DEFAULT public.generate_ulid() NOT NULL,
+    language_id uuid NOT NULL,
+    book_id integer,
+    chapters integer[],
+    layout text DEFAULT 'standard'::text NOT NULL,
+    status public.export_request_status DEFAULT 'PENDING'::public.export_request_status NOT NULL,
+    job_id uuid,
+    download_url text,
+    expires_at timestamp with time zone,
+    requested_by uuid NOT NULL,
+    requested_at timestamp with time zone DEFAULT now() NOT NULL,
+    completed_at timestamp with time zone,
+    export_key text
+);
+
+
+--
+-- Name: export_request_book; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.export_request_book (
+    request_id uuid NOT NULL,
+    book_id integer NOT NULL,
+    chapters integer[] NOT NULL
+);
+
 
 --
 -- Name: phrase; Type: TABLE; Schema: public; Owner: -
@@ -1196,6 +1238,16 @@ ALTER TABLE ONLY public.language_import_job
 ALTER TABLE ONLY public.language_member_role
     ADD CONSTRAINT language_member_role_pkey PRIMARY KEY (language_id, user_id, role);
 
+--
+-- Name: export_request export_request_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.export_request
+    ADD CONSTRAINT export_request_pkey PRIMARY KEY (id);
+
+ALTER TABLE ONLY public.export_request_book
+    ADD CONSTRAINT export_request_book_pkey PRIMARY KEY (request_id, book_id);
+
 
 --
 -- Name: language language_pkey; Type: CONSTRAINT; Schema: public; Owner: -
@@ -1456,6 +1508,18 @@ CREATE INDEX gloss_phrase_id_idx ON public.gloss USING btree (phrase_id);
 
 CREATE UNIQUE INDEX language_code_idx ON public.language USING btree (code);
 
+--
+-- Name: export_request_status_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX export_request_status_idx ON public.export_request USING btree (status);
+
+CREATE INDEX export_request_requested_by_idx ON public.export_request USING btree (requested_by);
+
+CREATE INDEX export_request_expires_at_idx ON public.export_request USING btree (expires_at);
+
+CREATE INDEX export_request_book_request_idx ON public.export_request_book USING btree (request_id);
+
 
 --
 -- Name: lemma_form_lemma_id_idx; Type: INDEX; Schema: public; Owner: -
@@ -1662,6 +1726,43 @@ ALTER TABLE ONLY public.language_member_role
 ALTER TABLE ONLY public.language_member_role
     ADD CONSTRAINT language_member_role_user_id_fkey FOREIGN KEY (user_id) REFERENCES public.users(id) ON UPDATE CASCADE ON DELETE RESTRICT;
 
+--
+-- Name: export_request export_request_book_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.export_request
+    ADD CONSTRAINT export_request_book_id_fkey FOREIGN KEY (book_id) REFERENCES public.book(id);
+
+
+--
+-- Name: export_request export_request_job_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.export_request
+    ADD CONSTRAINT export_request_job_id_fkey FOREIGN KEY (job_id) REFERENCES public.job(id);
+
+
+--
+-- Name: export_request export_request_language_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.export_request
+    ADD CONSTRAINT export_request_language_id_fkey FOREIGN KEY (language_id) REFERENCES public.language(id) ON DELETE CASCADE;
+
+
+--
+-- Name: export_request export_request_requested_by_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.export_request
+    ADD CONSTRAINT export_request_requested_by_fkey FOREIGN KEY (requested_by) REFERENCES public.users(id) ON DELETE CASCADE;
+
+ALTER TABLE ONLY public.export_request_book
+    ADD CONSTRAINT export_request_book_book_id_fkey FOREIGN KEY (book_id) REFERENCES public.book(id);
+
+ALTER TABLE ONLY public.export_request_book
+    ADD CONSTRAINT export_request_book_request_id_fkey FOREIGN KEY (request_id) REFERENCES public.export_request(id) ON DELETE CASCADE;
+
 
 --
 -- Name: language language_reference_language_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
@@ -1930,4 +2031,3 @@ ALTER TABLE ONLY public.word
 --
 -- PostgreSQL database dump complete
 --
-