Adding remove_duplicates to llm_util

redactor/config/default.yaml

@@ -6,39 +6,23 @@ redactors:
       model: "gpt-4.1"
       system_prompt: "You are a thorough assistant that extracts all of the requested terms from a given text."
       redaction_terms: 
-      - "People's names. List each part of the name separately."
-      constraints:
-      - "Do not include locations or organisations" 
-      - "Do not include names of anything which is not a person"
-      - "Do not list the author of the text"
-      - "Do not include those on whose behalf the text was written"
-
-    - name: "Text_Redactor_02"
-      model: "gpt-4.1"
-      system_prompt: "You are a thorough assistant that extracts all of the requested terms from a given text."
-      redaction_terms: 
-      - "Personal addresses and postcodes" 
+      - "People's names"
+      - "Personal addresses and postcodes"
       - "Personal email addresses, unless it's a Planning Inspectorate email"
       - "Telephone numbers, unless it's a Planning Inspectorate customer service team telephone number"
       - "National Insurance Numbers, e.g. AB 12 34 56 C"
       - "Hyperlinks, except those that are .gov.uk, .org, .gov.wales"
-
-    - name: "Text_Redactor_03"
-      model: "gpt-4.1"
-      system_prompt: "You are a thorough assistant that extracts all of the requested terms from a given text."
-      redaction_terms: 
       - "Personal health information, e.g. illnesses or concerning a person's sex life. List each term as it appears in the text."
       - "Personal data revealing ethnic origin, political opinions, philosophical beliefs, or trade union membership"
       - "Criminal offence data, e.g. allegations, investigations, proceedings, penalties"
       - "Any defamatory (libellous) or inflammatory information"
       - "Specific financial information such as bank accounts, salary details, house valuations, bonuses, or shares"
-
-    - name: "Text_Redactor_04"
-      model: "gpt-4.1"
-      system_prompt: "You are a thorough assistant that extracts all of the requested terms from a given text."
-      redaction_terms: 
       - "Dates of birth, and ages of people"
       - "The location of any of the following: badger sett, bat maternity roost, bird nest"
+      constraints:
+      - "Do not include names of anything which is not a person"
+      - "Do not include the name of the author of the text"
+      - "Do not include the names of those on whose behalf the text was written"
 
   - redactor_type: "ImageRedaction"
 
@@ -55,14 +39,5 @@ redactors:
     redaction_rules: 
     - name: "Image_Text_Redactor_01"
       text_redaction_rule: "Text_Redactor_01"
-
-    - name: "Image_Text_Redactor_02"
-      text_redaction_rule: "Text_Redactor_02"
-
-    - name: "Image_Text_Redactor_03"
-      text_redaction_rule: "Text_Redactor_03"
-
-    - name: "Image_Text_Redactor_04"
-      text_redaction_rule: "Text_Redactor_04"
 
 provisional_redactions:

redactor/config/stopwords.yaml

@@ -0,0 +1,3 @@
+stopwords:
+  - "the"
+  - "my"

redactor/core/io/azure_blob_io.py

@@ -4,6 +4,8 @@
     ChainedTokenCredential,
 )
 from azure.storage.blob import BlobServiceClient
+from azure.core.exceptions import ResourceExistsError
+from core.util.logging_util import LoggingUtil
 from io import BytesIO
 from typing import Any
 from .storage_io import StorageIO
@@ -35,6 +37,9 @@ def get_kind(cls):
         return "AzureBlob"
 
     def read(self, container_name: str, blob_path: str, **kwargs) -> BytesIO:
+        LoggingUtil().log_info(
+            f"Reading blob '{blob_path}' from container '{container_name}' in storage account '{self.storage_endpoint}'"
+        )
         blob_service_client = BlobServiceClient(
             self.storage_endpoint, credential=self.credential
         )
@@ -45,10 +50,19 @@ def read(self, container_name: str, blob_path: str, **kwargs) -> BytesIO:
         return byte_stream
 
     def write(self, data_bytes: BytesIO, container_name: str, blob_path: str, **kwargs):
+        LoggingUtil().log_info(
+            f"Writing blob '{blob_path}' from container '{container_name}' in storage account '{self.storage_endpoint}'"
+        )
         blob_service_client = BlobServiceClient(
             self.storage_endpoint, credential=self.credential
         )
         blob_client = blob_service_client.get_blob_client(
             container=container_name, blob=blob_path
         )
-        blob_client.upload_blob(data_bytes, blob_type="BlockBlob")
+        try:
+            blob_client.upload_blob(data_bytes, blob_type="BlockBlob")
+        except ResourceExistsError:
+            # Improve the base Azure error, which does not include helpful info
+            raise ResourceExistsError(
+                f"The specified blob {self.storage_endpoint}/{container_name}/{blob_path} already exists"
+            )