diff --git a/addons/hdfs-operator/README.md b/addons/hdfs-operator/README.md new file mode 100644 index 0000000..208fbbc --- /dev/null +++ b/addons/hdfs-operator/README.md @@ -0,0 +1,7 @@ +# Kubedoop Operator for Apache Hadoop + +This is a Kubernetes operator to manage Apache Hadoop HDFS clusters. + +It's part of the kubedoop Data Platform, a modular open source data platform built on Kubernetes that provides Kubernetes native deployment +and management of popular open source data apps like Apache Kafka, Apache Doris, Apache Kyuubi, Trino or Apache Spark, all working +together seamlessly. Based on Kubernetes, it runs everywhere – on prem or in the cloud. diff --git a/addons/hdfs-operator/metadata.yaml b/addons/hdfs-operator/metadata.yaml new file mode 100644 index 0000000..d4bc02f --- /dev/null +++ b/addons/hdfs-operator/metadata.yaml @@ -0,0 +1,16 @@ +name: hdfs-operator +version: 0.1.0 +description: Apache Hadoop HDFS operator for the Kubedoop. +icon: https://github.com/zncdatadev/kubedoop/raw/main/docs/assets/icon_kubedoop.png +url: https://github.com/zncdatadev/hdfs-operator + +tags: + - kubedoop + +dependencies: + - name: commons-operator + - name: listener-operator + - name: secret-operator + +system: + vela: ">=v1.9.0" diff --git a/addons/hdfs-operator/parameter.cue b/addons/hdfs-operator/parameter.cue new file mode 100644 index 0000000..45a6d7f --- /dev/null +++ b/addons/hdfs-operator/parameter.cue @@ -0,0 +1,10 @@ +parameter: { + // +usage=Specify if upgrade the CRDs when upgrading ingress-nginx or not + upgradeCRD: *false | bool + //+usage=Deploy to specified clusters. Leave empty to deploy to all clusters. + clusters?: [...string] + //+usage=Namespace to deploy to, defaults to kubedoop-system + namespace: *"kubedoop-system" | string + //+usage=Number of replicas + replicas: *1 | int +} diff --git a/addons/hdfs-operator/resources/hdfs-operator.cue b/addons/hdfs-operator/resources/hdfs-operator.cue new file mode 100644 index 0000000..7c43735 --- /dev/null +++ b/addons/hdfs-operator/resources/hdfs-operator.cue @@ -0,0 +1,19 @@ +package main + +hdfsOperator: { + name: "hdfs-operator" + type: "helm" + dependsOn: ["hdfs-operator-ns"] + properties: { + repoType: "helm" + url: "https://zncdatadev.github.io/kubedoop-helm-charts" + chart: "hdfs-operator" + releaseName: "hdfs-operator" + targetNamespace: parameter.namespace + version: context.metadata.version + upgradeCRD: parameter.upgradeCRD + values: { + replicaCount: parameter.replicas + } + } +} diff --git a/addons/hdfs-operator/resources/topology.cue b/addons/hdfs-operator/resources/topology.cue new file mode 100644 index 0000000..a9792fc --- /dev/null +++ b/addons/hdfs-operator/resources/topology.cue @@ -0,0 +1,32 @@ +package main + +import "encoding/json" + +resourceTopology: { + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "hdfs-operator-topology" + namespace: "vela-system" + labels: { + "rules.oam.dev/resources": "true" + "rules.oam.dev/resource-format": "json" + } + } + data: rules: json.Marshal([{ + parentResourceType: { + group: "hdfs.kubedoop.dev" + kind: "HdfsCluster" + } + childrenResourceType: [ + { + apiVersion: "apps/v1" + kind: "StatefulSet" + }, + { + apiVersion: "v1" + kind: "Service" + }, + ] + }]) +} \ No newline at end of file diff --git a/addons/hdfs-operator/template.cue b/addons/hdfs-operator/template.cue new file mode 100644 index 0000000..13060af --- /dev/null +++ b/addons/hdfs-operator/template.cue @@ -0,0 +1,46 @@ +package main + +_targetNamespace: parameter.namespace + +output: { + apiVersion: "core.oam.dev/v1beta1" + kind: "Application" + spec: { + components: [ + { + type: "k8s-objects" + name: "hdfs-operator-ns" + properties: objects: [{ + apiVersion: "v1" + kind: "Namespace" + metadata: name: _targetNamespace + }] + }, + hdfsOperator + ] + policies: [ + { + type: "shared-resource" + name: "hdfs-operator-ns" + properties: rules: [{ + selector: resourceTypes: ["Namespace"] + }] + }, + { + type: "topology" + name: "deploy-topology" + properties: { + namespace: _targetNamespace + if parameter.clusters != _|_ { + clusters: parameter.clusters + } + if parameter.clusters == _|_ { + clusterLabelSelector: {} + } + } + }, + ] + } +} + +outputs: topology: resourceTopology diff --git a/examples/hdfs-operator/hdfs-cluster.yaml b/examples/hdfs-operator/hdfs-cluster.yaml new file mode 100644 index 0000000..46ebcc3 --- /dev/null +++ b/examples/hdfs-operator/hdfs-cluster.yaml @@ -0,0 +1,42 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: hdfs-cluster +spec: + components: + - name: "hdfs-cluster" + type: "k8s-objects" + properties: + objects: + - apiVersion: hdfs.kubedoop.dev/v1alpha1 + kind: HdfsCluster + metadata: + name: simple-hdfs + spec: + image: + productVersion: 3.4.0 + clusterConfig: + zookeeperConfigMapName: simple-hdfs-znode + dfsReplication: 1 + nameNodes: + config: + listenerClass: external-stable + roleGroups: + default: + replicas: 2 + dataNodes: + config: + listenerClass: external-unstable + roleGroups: + default: + replicas: 1 + journalNodes: + roleGroups: + default: + replicas: 1 + policies: + - name: topology-demo + type: topology + properties: + clusters: ['local'] + namespace: demo \ No newline at end of file diff --git a/examples/hdfs-operator/zookeeper-cluster.yaml b/examples/hdfs-operator/zookeeper-cluster.yaml new file mode 100644 index 0000000..9436d7f --- /dev/null +++ b/examples/hdfs-operator/zookeeper-cluster.yaml @@ -0,0 +1,34 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: zookeeper-cluster +spec: + components: + - name: "zookeeper-cluster" + type: "k8s-objects" + properties: + objects: + - apiVersion: zookeeper.kubedoop.dev/v1alpha1 + kind: ZookeeperCluster + metadata: + name: simple-zk + spec: + image: + productVersion: 3.9.2 + servers: + roleGroups: + default: + replicas: 1 + - apiVersion: zookeeper.kubedoop.dev/v1alpha1 + kind: ZookeeperZnode + metadata: + name: simple-hdfs-znode + spec: + clusterRef: + name: simple-zk + policies: + - name: topology-demo + type: topology + properties: + clusters: ['local'] + namespace: demo \ No newline at end of file