From 4404875c9b99c8bd1d8ac20537465fbc5de2a30d Mon Sep 17 00:00:00 2001 From: jihooncho Date: Fri, 7 Oct 2022 02:14:26 +0900 Subject: [PATCH 1/3] add kerberos auth from keytab --- build.gradle | 2 +- .../output/hdfs/HdfsFileOutputPlugin.java | 12 ++++++++++ .../embulk/output/hdfs/client/HdfsClient.java | 22 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2948563..308eada 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ configurations { provided } -version = "0.4.0.pre" +version = "0.4.1.pre" sourceCompatibility = 1.8 targetCompatibility = 1.8 diff --git a/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java b/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java index 437dced..15e4df9 100644 --- a/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java +++ b/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java @@ -59,6 +59,18 @@ public interface PluginTask @ConfigDefault("null") Optional getDoas(); + @Config("krb5_config_path") + @ConfigDefault("null") + Optional getKrb5ConfigPath(); + + @Config("keytab_principal") + @ConfigDefault("null") + Optional getKeytabPrincipal(); + + @Config("keytab_path") + @ConfigDefault("null") + Optional getKeytabPath(); + @Deprecated enum DeleteInAdvancePolicy { diff --git a/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java b/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java index 9613e8c..457abc5 100644 --- a/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java +++ b/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java @@ -17,6 +17,7 @@ import org.slf4j.Logger; import java.io.File; +import java.io.IOException; import java.io.OutputStream; import java.net.MalformedURLException; import java.net.URI; @@ -27,12 +28,33 @@ public class HdfsClient { public static HdfsClient build(HdfsFileOutputPlugin.PluginTask task) { + setKerberosKeytabAuthention(task); Configuration conf = buildConfiguration(task.getConfigFiles(), task.getConfig()); return new HdfsClient(conf, task.getDoas()); } ; + /** + * https://docs.cloudera.com/documentation/enterprise/6/6.2/topics/cdh_sg_princ_auth_java.html + */ + public static void setKerberosKeytabAuthention(HdfsFileOutputPlugin.PluginTask task){ + if(task.getKeytabPath().isPresent() && + task.getKeytabPrincipal().isPresent() && + task.getKrb5ConfigPath().isPresent() + ){ + logger.info("Kerberos Keytab conf = krb5Path :{}, keytabprincipal: {}, keytabPath: {}", task.getKrb5ConfigPath().get(), task.getKeytabPrincipal().get(), task.getKeytabPath().get()); + try { + System.setProperty("java.security.krb5.conf", task.getKrb5ConfigPath().get()); + UserGroupInformation.loginUserFromKeytab(task.getKeytabPrincipal().get(), task.getKeytabPath().get()); + } catch (IOException e) { + throw new ConfigException(e); + } + } + } + + ; + public static Configuration buildConfiguration(List configFiles, Map configs) { Configuration c = new Configuration(); From 0362c3b6ae284538e0c48ec36c003dbaa4aa85df Mon Sep 17 00:00:00 2001 From: jihooncho Date: Fri, 7 Oct 2022 02:37:21 +0900 Subject: [PATCH 2/3] add kerberos auth from keytab to config --- README.md | 4 ++ .../output/hdfs/HdfsFileOutputPlugin.java | 15 ++------ .../embulk/output/hdfs/client/HdfsClient.java | 37 ++++++++++++------- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index a834942..af880a2 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,10 @@ A File Output Plugin for Embulk to write HDFS. - `RECURSIVE`: delete files and directories - **mode**: "abort_if_exist", "overwrite", "delete_files_in_advance", "delete_recursive_in_advance", or "replace". See below. (string, optional, default: `"abort_if_exist"`) * In the future, default mode will become `"replace"`. +- **keytab_config**: For keytab auth for kerberos. see https://docs.cloudera.com/documentation/enterprise/6/6.2/topics/cdh_sg_princ_auth_java.html (hash, default: `{}`) + - **krb5_config_path**: krb5.conf file path (string, optional) + - **keytab_principal**: user principal (string, optional) + - **keytab_path**: Keytab file path (string, optional) ## CAUTION If you use `hadoop` user (hdfs admin user) as `doas`, and if `delete_in_advance` is `RECURSIVE`, diff --git a/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java b/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java index 15e4df9..000162b 100644 --- a/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java +++ b/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java @@ -59,18 +59,9 @@ public interface PluginTask @ConfigDefault("null") Optional getDoas(); - @Config("krb5_config_path") - @ConfigDefault("null") - Optional getKrb5ConfigPath(); - - @Config("keytab_principal") - @ConfigDefault("null") - Optional getKeytabPrincipal(); - - @Config("keytab_path") - @ConfigDefault("null") - Optional getKeytabPath(); - + @Config("keytab_config") + @ConfigDefault("{}") + Map getKeytabConfig(); @Deprecated enum DeleteInAdvancePolicy { diff --git a/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java b/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java index 457abc5..9f5f32e 100644 --- a/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java +++ b/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java @@ -1,6 +1,7 @@ package org.embulk.output.hdfs.client; import com.google.common.base.Optional; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; @@ -28,7 +29,7 @@ public class HdfsClient { public static HdfsClient build(HdfsFileOutputPlugin.PluginTask task) { - setKerberosKeytabAuthention(task); + setKerberosKeytabAuthention(task.getKeytabConfig()); Configuration conf = buildConfiguration(task.getConfigFiles(), task.getConfig()); return new HdfsClient(conf, task.getDoas()); } @@ -38,19 +39,29 @@ public static HdfsClient build(HdfsFileOutputPlugin.PluginTask task) /** * https://docs.cloudera.com/documentation/enterprise/6/6.2/topics/cdh_sg_princ_auth_java.html */ - public static void setKerberosKeytabAuthention(HdfsFileOutputPlugin.PluginTask task){ - if(task.getKeytabPath().isPresent() && - task.getKeytabPrincipal().isPresent() && - task.getKrb5ConfigPath().isPresent() - ){ - logger.info("Kerberos Keytab conf = krb5Path :{}, keytabprincipal: {}, keytabPath: {}", task.getKrb5ConfigPath().get(), task.getKeytabPrincipal().get(), task.getKeytabPath().get()); - try { - System.setProperty("java.security.krb5.conf", task.getKrb5ConfigPath().get()); - UserGroupInformation.loginUserFromKeytab(task.getKeytabPrincipal().get(), task.getKeytabPath().get()); - } catch (IOException e) { - throw new ConfigException(e); - } + public static void setKerberosKeytabAuthention(Map keytabConfig){ + if(keytabConfig == null || keytabConfig.size() == 0){ + return; + } + String krb5ConfigPath = keytabConfig.get("krb5_config_path"); + String keytabPrincipal = keytabConfig.get("keytab_principal"); + String keytabPath = keytabConfig.get("keytab_path"); + + logger.info("Keytab config init. krb5_config_path: {} , keytab_principal : {}, keytab_path : {}", krb5ConfigPath, keytabPrincipal, keytabPath); + + if(StringUtils.isEmpty(krb5ConfigPath) || + StringUtils.isEmpty(keytabPrincipal) || + StringUtils.isEmpty(keytabPath)){ + throw new ConfigException(String.format("Keytab config not enough. krb5_config_path: {} , keytab_principal : {}, keytab_path : {}", krb5ConfigPath, keytabPrincipal, keytabPath)); } + + try { + System.setProperty("java.security.krb5.conf", krb5ConfigPath); + UserGroupInformation.loginUserFromKeytab(keytabPrincipal, keytabPath); + } catch (IOException e) { + throw new ConfigException(e); + } + } ; From c55d8caf1a05612ec39ce9afaeb3342a25820e42 Mon Sep 17 00:00:00 2001 From: jihooncho Date: Fri, 7 Oct 2022 06:55:51 +0900 Subject: [PATCH 3/3] keytab auth --- build.gradle | 2 +- .../java/org/embulk/output/hdfs/client/HdfsClient.java | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 308eada..94fc532 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ configurations { provided } -version = "0.4.1.pre" +version = "0.4.2.pre" sourceCompatibility = 1.8 targetCompatibility = 1.8 diff --git a/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java b/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java index 9f5f32e..24ab29c 100644 --- a/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java +++ b/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java @@ -29,8 +29,8 @@ public class HdfsClient { public static HdfsClient build(HdfsFileOutputPlugin.PluginTask task) { - setKerberosKeytabAuthention(task.getKeytabConfig()); Configuration conf = buildConfiguration(task.getConfigFiles(), task.getConfig()); + setKerberosKeytabAuthention(conf, task.getKeytabConfig()); return new HdfsClient(conf, task.getDoas()); } @@ -39,8 +39,9 @@ public static HdfsClient build(HdfsFileOutputPlugin.PluginTask task) /** * https://docs.cloudera.com/documentation/enterprise/6/6.2/topics/cdh_sg_princ_auth_java.html */ - public static void setKerberosKeytabAuthention(Map keytabConfig){ + public static void setKerberosKeytabAuthention(Configuration conf, Map keytabConfig){ if(keytabConfig == null || keytabConfig.size() == 0){ + UserGroupInformation.setConfiguration(conf); return; } String krb5ConfigPath = keytabConfig.get("krb5_config_path"); @@ -57,6 +58,7 @@ public static void setKerberosKeytabAuthention(Map keytabConfig) try { System.setProperty("java.security.krb5.conf", krb5ConfigPath); + UserGroupInformation.setConfiguration(conf); UserGroupInformation.loginUserFromKeytab(keytabPrincipal, keytabPath); } catch (IOException e) { throw new ConfigException(e); @@ -81,7 +83,6 @@ public static Configuration buildConfiguration(List configFiles, Map config : configs.entrySet()) { c.set(config.getKey(), config.getValue()); } - UserGroupInformation.setConfiguration(c); return c; }