Metadata feature #554

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

romanost03 wants to merge 70 commits into polypheny:master from romanost03:metadataFeature

build.gradle

-Original file line number
+Diff line change
@@ Expand Up / @@ -144,6 +144,8 @@ allprojects { @@
             implementation group: "org.slf4j", name: "slf4j-api", version: slf4j_api_version  // MIT
             implementation group: "org.apache.logging.log4j", name: "log4j-slf4j2-impl", version: log4j_slf4j_impl_version  // Apache 2.0
+            implementation 'org.postgresql:postgresql:42.7.2'
+            implementation group: 'com.oracle.database.jdbc', name: 'ojdbc8', version: '19.8.0.0'
             testCompileOnly(group: 'org.pf4j', name: 'pf4j', version: pf4j_version) {
                 exclude group: "org.slf4j"
@@ Expand Down @@

core/src/main/java/org/polypheny/db/adapter/AdapterManager.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -19,13 +19,18 @@ @@
     import com.google.common.collect.ImmutableMap;
     import com.google.gson.JsonObject;
     import com.google.gson.JsonSerializer;
+    import java.io.IOException;
     import java.lang.reflect.Modifier;
+    import java.nio.file.Files;
+    import java.nio.file.Path;
+    import java.nio.file.Paths;
     import java.util.ArrayList;
     import java.util.HashMap;
     import java.util.List;
     import java.util.Map;
     import java.util.Map.Entry;
     import java.util.Optional;
+    import lombok.extern.slf4j.Slf4j;
     import org.apache.calcite.linq4j.tree.Expression;
     import org.apache.calcite.linq4j.tree.Expressions;
     import org.jetbrains.annotations.NotNull;
@@ Expand All / @@ -37,8 +42,10 @@ @@
     import org.polypheny.db.catalog.entity.LogicalAdapter.AdapterType;
     import org.polypheny.db.catalog.entity.allocation.AllocationEntity;
     import org.polypheny.db.catalog.exceptions.GenericRuntimeException;
+    import org.polypheny.db.schemaDiscovery.MetadataProvider;
+    @Slf4j
     public class AdapterManager {
         public static Expression ADAPTER_MANAGER_EXPRESSION = Expressions.call( AdapterManager.class, "getInstance" );
@@ Expand Down Expand Up @@
             AdapterTemplate adapterTemplate = AdapterTemplate.fromString( adapterName, adapterType );
             for ( AbstractAdapterSetting setting : adapterTemplate.settings ) {
                 if ( setting.appliesTo.stream().noneMatch( s -> s.appliesTo( mode ) ) ) {
                     settings.remove( setting.name );
@@ Expand Down Expand Up / @@ -217,6 +223,30 @@ public void removeAdapter( long adapterId ) { @@
             // Shutdown store
             adapterInstance.shutdownAndRemoveListeners();
+            // Delete directory if exist
+            try {
+                var s = adapterInstance.getCurrentSettings();
+                if ( s != null && s.containsKey( "directory" ) ) {
+                    String dirStr = s.get( "directory" );
+                    if ( dirStr != null && !dirStr.isBlank() ) {
+                        Path dir = Paths.get( dirStr ).normalize();
+                        if ( Files.isDirectory( dir ) ) {
+                            Files.walk( dir )
+                                    .sorted( java.util.Comparator.reverseOrder() )
+                                    .forEach( p -> {
+                                        try {
+                                            Files.deleteIfExists( p );
+                                        } catch ( IOException ignored ) {
+                                        }
+                                    } );
+                            log.info( "Deleted adapter directory: {}", dir.toAbsolutePath().toString() );
+                        }
+                    }
+                }
+            } catch ( Exception e ) {
+                log.warn( "Could not delete adapter directory: {}", e.toString() );
+            }
             // Remove store from maps
             adapterById.remove( adapterInstance.getAdapterId() );
             adapterByName.remove( adapterInstance.getUniqueName() );
@@ Expand All @@
         }
+        public Optional<MetadataProvider> getMetadataProvider( String uniqueName ) {
+            return getSource( uniqueName )
+                    .filter( mp -> mp instanceof MetadataProvider )
+                    .map( mp -> (MetadataProvider) mp );
+        }
         public record AdapterInformation( String name, String description, AdapterType type, List<AbstractAdapterSetting> settings, List<DeployMode> modes ) {
             public static JsonSerializer<AdapterInformation> getSerializer() {
@@ Expand Down @@

core/src/main/java/org/polypheny/db/adapter/DataSource.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -25,6 +25,8 @@ @@
     import org.polypheny.db.catalog.catalogs.AdapterCatalog;
     import org.polypheny.db.catalog.entity.LogicalAdapter.AdapterType;
     import org.polypheny.db.catalog.logistic.DataModel;
+    import org.polypheny.db.schemaDiscovery.MetadataProvider;
+    import org.polypheny.db.type.PolyType;
     @Getter
     public abstract class DataSource<S extends AdapterCatalog> extends Adapter<S> implements ExtensionPoint {
@@ Expand Down @@

core/src/main/java/org/polypheny/db/adapter/MetadataObserver/AbstractListener.java

-Original file line number
+Diff line change
@@ -0,0 +1,31 @@
+    /*
+     * Copyright 2019-2025 The Polypheny Project
+     *
+     * Licensed under the Apache License, Version 2.0 (the "License");
+     * you may not use this file except in compliance with the License.
+     * You may obtain a copy of the License at
+     *
+     * http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+    package org.polypheny.db.adapter.MetadataObserver;
+    import org.polypheny.db.adapter.Adapter;
+    import org.polypheny.db.schemaDiscovery.AbstractNode;
+    import org.polypheny.db.schemaDiscovery.MetadataProvider;
+    public interface AbstractListener<P extends Adapter & MetadataProvider> {
+        void onMetadataChange( P adapter, AbstractNode node, String hash );
+        boolean isAvailable();
+        void applyChange( String[] metadata );
+    }

core/src/main/java/org/polypheny/db/adapter/MetadataObserver/AbstractPublisher.java

-Original file line number
+Diff line change
@@ -0,0 +1,32 @@
+    /*
+     * Copyright 2019-2025 The Polypheny Project
+     *
+     * Licensed under the Apache License, Version 2.0 (the "License");
+     * you may not use this file except in compliance with the License.
+     * You may obtain a copy of the License at
+     *
+     * http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+    package org.polypheny.db.adapter.MetadataObserver;
+    public interface AbstractPublisher {
+        String getAdapterUniqueName();
+        void start();
+        void stop();
+        void runCheck();
+        AbstractListener getListener();
+    }

core/src/main/java/org/polypheny/db/adapter/MetadataObserver/ChangeLogEntry.java

-Original file line number
+Diff line change
@@ -0,0 +1,66 @@
+    /*
+     * Copyright 2019-2025 The Polypheny Project
+     *
+     * Licensed under the Apache License, Version 2.0 (the "License");
+     * you may not use this file except in compliance with the License.
+     * You may obtain a copy of the License at
+     *
+     * http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+    package org.polypheny.db.adapter.MetadataObserver;
+    import com.fasterxml.jackson.annotation.JsonProperty;
+    import lombok.AllArgsConstructor;
+    import lombok.Value;
+    import org.polypheny.db.adapter.MetadataObserver.PublisherManager.ChangeStatus;
+    import org.polypheny.db.adapter.MetadataObserver.Utils.MetaDiffUtil.DiffResult;
+    import java.time.Instant;
+    import java.util.ArrayList;
+    import java.util.List;
+    @Value
+    @AllArgsConstructor
+    public class ChangeLogEntry {
+        @JsonProperty
+        String adapterName;
+        @JsonProperty
+        Instant timestamp;
+        @JsonProperty
+        List<String> messages;
+        @JsonProperty
+        ChangeStatus severity;
+        public class DiffMessageUtil {
+            private DiffMessageUtil() {
+            }
+            public static List<String> toMessages( DiffResult diff ) {
+                List<String> msgs = new ArrayList<>();
+                diff.getAdded()
+                        .forEach( p -> msgs.add( "Added metadata " + p ) );
+                diff.getRemoved()
+                        .forEach( p -> msgs.add( "Removed metadata " + p ) );
+                diff.getChanged()
+                        .forEach( p -> msgs.add( "Changed metadata " + p ) );
+                return msgs;
+            }
+        }
+    }

core/src/main/java/org/polypheny/db/adapter/MetadataObserver/ChangeLogView.java

-Original file line number
+Diff line change
@@ -0,0 +1,88 @@
+    /*
+     * Copyright 2019-2025 The Polypheny Project
+     *
+     * Licensed under the Apache License, Version 2.0 (the "License");
+     * you may not use this file except in compliance with the License.
+     * You may obtain a copy of the License at
+     *
+     * http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+    package org.polypheny.db.adapter.MetadataObserver;
+    import com.fasterxml.jackson.annotation.JsonProperty;
+    import lombok.AllArgsConstructor;
+    import lombok.Value;
+    import org.polypheny.db.adapter.MetadataObserver.PublisherManager.ChangeStatus;
+    import java.time.Duration;
+    import java.time.Instant;
+    import java.util.List;
+    @Value
+    @AllArgsConstructor
+    public class ChangeLogView {
+        @JsonProperty
+        String adapterName;
+        @JsonProperty
+        String timestamp;
+        @JsonProperty
+        List<String> messages;
+        @JsonProperty
+        ChangeStatus severity;
+        public static ChangeLogView from( ChangeLogEntry e ) {
+            return new ChangeLogView(
+                    e.getAdapterName(),
+                    rel( e.getTimestamp() ),
+                    e.getMessages(),
+                    e.getSeverity()
+            );
+        }
+        private static String rel( Instant then ) {
+            long s = Duration.between( then, Instant.now() ).getSeconds();
+            if ( s < 0 ) {
+                s = 0;
+            }
+            if ( s < 60 ) {
+                return s == 1 ? "1 second ago" : s + " seconds ago";
+            }
+            long m = s / 60;
+            if ( m < 60 ) {
+                return m == 1 ? "1 minute ago" : m + " minutes ago";
+            }
+            long h = m / 60;
+            if ( h < 24 ) {
+                return h == 1 ? "1 hour ago" : h + " hours ago";
+            }
+            long d = h / 24;
+            if ( d < 7 ) {
+                return d == 1 ? "1 day ago" : d + " days ago";
+            }
+            long w = d / 7;
+            if ( w < 5 ) {
+                return w == 1 ? "1 week ago" : w + " weeks ago";
+            }
+            long mo = d / 30;
+            if ( mo < 12 ) {
+                return mo == 1 ? "1 month ago" : mo + " months ago";
+            }
+            long y = d / 365;
+            return y == 1 ? "1 year ago" : y + " years ago";
+        }
+    }

core/src/main/java/org/polypheny/db/adapter/MetadataObserver/HashCache.java

-Original file line number
+Diff line change
@@ -0,0 +1,48 @@
+    /*
+     * Copyright 2019-2025 The Polypheny Project
+     *
+     * Licensed under the Apache License, Version 2.0 (the "License");
+     * you may not use this file except in compliance with the License.
+     * You may obtain a copy of the License at
+     *
+     * http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+    package org.polypheny.db.adapter.MetadataObserver;
+    import java.util.Map;
+    import java.util.concurrent.ConcurrentHashMap;
+    public class HashCache {
+        private static final HashCache INSTANCE = new HashCache();
+        private final Map<String, String> cache = new ConcurrentHashMap<>();
+        private HashCache() {
+        }
+        public static HashCache getInstance() {
+            return INSTANCE;
+        }
+        public void put( String uniqueName, String hash ) {
+            this.cache.put( uniqueName, hash );
+        }
+        public String getHash( String uniqueName ) {
+            return this.cache.get( uniqueName );
+        }
+    }

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Metadata feature #554

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Metadata feature #554

Are you sure you want to change the base?

Uh oh!

Metadata feature #554

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!