Skip to content

Commit 32e75b2

Browse files
bbrownzbbrownz
committed
CDK-476 oozie UriHandler for Kite URIs
1 parent 78e9fa7 commit 32e75b2

File tree

5 files changed

+790
-1
lines changed

5 files changed

+790
-1
lines changed

‎kite-data/kite-data-oozie/pom.xml‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Copyright 2015 Cloudera Inc.
4+
~
5+
~ Licensed under the Apache License, Version 2.0 (the "License");
6+
~ you may not use this file except in compliance with the License.
7+
~ You may obtain a copy of the License at
8+
~
9+
~ http://www.apache.org/licenses/LICENSE-2.0
10+
~
11+
~ Unless required by applicable law or agreed to in writing, software
12+
~ distributed under the License is distributed on an "AS IS" BASIS,
13+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
~ See the License for the specific language governing permissions and
15+
~ limitations under the License.
16+
-->
17+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
18+
19+
<modelVersion>4.0.0</modelVersion>
20+
<artifactId>kite-data-oozie</artifactId>
21+
22+
<parent>
23+
<groupId>org.kitesdk</groupId>
24+
<artifactId>kite-data</artifactId>
25+
<version>1.0.1-SNAPSHOT</version>
26+
</parent>
27+
28+
<name>Kite Data Oozie Module</name>
29+
<description>
30+
The Kite Data Oozie module provides Oozie support for working with Kite datasets.
31+
</description>
32+
33+
<properties>
34+
<vers.oozie4>4.1.0</vers.oozie4>
35+
</properties>
36+
37+
<build>
38+
<plugins>
39+
<plugin>
40+
<groupId>org.apache.maven.plugins</groupId>
41+
<artifactId>maven-compiler-plugin</artifactId>
42+
</plugin>
43+
<plugin>
44+
<groupId>org.apache.maven.plugins</groupId>
45+
<artifactId>maven-source-plugin</artifactId>
46+
</plugin>
47+
<plugin>
48+
<groupId>org.apache.maven.plugins</groupId>
49+
<artifactId>maven-javadoc-plugin</artifactId>
50+
<configuration>
51+
<excludePackageNames>org.kitesdk.data.oozie</excludePackageNames>
52+
</configuration>
53+
</plugin>
54+
<plugin>
55+
<groupId>org.apache.maven.plugins</groupId>
56+
<artifactId>maven-surefire-plugin</artifactId>
57+
</plugin>
58+
<plugin>
59+
<groupId>org.apache.rat</groupId>
60+
<artifactId>apache-rat-plugin</artifactId>
61+
</plugin>
62+
<plugin>
63+
<groupId>org.codehaus.mojo</groupId>
64+
<artifactId>findbugs-maven-plugin</artifactId>
65+
</plugin>
66+
<plugin>
67+
<groupId>org.apache.maven.plugins</groupId>
68+
<artifactId>maven-antrun-plugin</artifactId>
69+
<executions>
70+
<execution>
71+
<phase>compile</phase>
72+
<goals>
73+
<goal>run</goal>
74+
</goals>
75+
<configuration>
76+
<tasks>
77+
<echo message="Create empty javadoc JAR to satisfy Maven central" />
78+
<mkdir dir="target/apidocs" />
79+
</tasks>
80+
</configuration>
81+
</execution>
82+
</executions>
83+
</plugin>
84+
</plugins>
85+
</build>
86+
87+
<reporting>
88+
<plugins>
89+
<plugin>
90+
<groupId>org.apache.maven.plugins</groupId>
91+
<artifactId>maven-project-info-reports-plugin</artifactId>
92+
<reportSets>
93+
<reportSet>
94+
<inherited>false</inherited>
95+
<reports>
96+
<report>index</report>
97+
<report>summary</report>
98+
<report>dependency-info</report>
99+
<report>dependencies</report>
100+
</reports>
101+
</reportSet>
102+
</reportSets>
103+
</plugin>
104+
<plugin>
105+
<groupId>org.apache.maven.plugins</groupId>
106+
<artifactId>maven-javadoc-plugin</artifactId>
107+
</plugin>
108+
</plugins>
109+
</reporting>
110+
111+
<dependencies>
112+
<!-- Kite -->
113+
<dependency>
114+
<groupId>org.kitesdk</groupId>
115+
<artifactId>kite-data-core</artifactId>
116+
<version>${project.parent.version}</version>
117+
</dependency>
118+
119+
<!-- Hadoop -->
120+
<dependency>
121+
<groupId>org.kitesdk</groupId>
122+
<artifactId>${artifact.hadoop-deps}</artifactId>
123+
<type>pom</type>
124+
<scope>provided</scope>
125+
<exclusions>
126+
<exclusion>
127+
<!-- conflicts with jetty included with HBase -->
128+
<groupId>tomcat</groupId>
129+
<artifactId>jasper-runtime</artifactId>
130+
</exclusion>
131+
</exclusions>
132+
</dependency>
133+
<dependency>
134+
<groupId>org.kitesdk</groupId>
135+
<artifactId>kite-hadoop-compatibility</artifactId>
136+
</dependency>
137+
138+
<!-- custom URI handlers require Oozie 4.x -->
139+
<dependency>
140+
<groupId>org.apache.oozie</groupId>
141+
<artifactId>oozie-core</artifactId>
142+
<version>${vers.oozie4}</version>
143+
</dependency>
144+
145+
<!-- Misc -->
146+
<dependency>
147+
<groupId>com.google.code.findbugs</groupId>
148+
<artifactId>jsr305</artifactId>
149+
<scope>provided</scope>
150+
<optional>true</optional>
151+
</dependency>
152+
<dependency>
153+
<groupId>com.google.code.findbugs</groupId>
154+
<artifactId>annotations</artifactId>
155+
<scope>provided</scope>
156+
</dependency>
157+
158+
<!-- Test Dependencies -->
159+
<dependency>
160+
<groupId>junit</groupId>
161+
<artifactId>junit</artifactId>
162+
<scope>test</scope>
163+
</dependency>
164+
<dependency>
165+
<groupId>org.kitesdk</groupId>
166+
<artifactId>kite-data-core</artifactId>
167+
<version>${project.parent.version}</version>
168+
<type>test-jar</type>
169+
<scope>test</scope>
170+
</dependency>
171+
<dependency>
172+
<groupId>org.kitesdk</groupId>
173+
<artifactId>${artifact.hadoop-test-deps}</artifactId>
174+
<type>pom</type>
175+
<scope>test</scope>
176+
</dependency>
177+
</dependencies>
178+
179+
<profiles>
180+
<profile>
181+
<id>cdh5</id>
182+
<activation>
183+
<property>
184+
<name>hadoop.profile</name>
185+
<value>cdh5</value>
186+
</property>
187+
</activation>
188+
<properties>
189+
<vers.oozie4>${vers.oozie}</vers.oozie4>
190+
</properties>
191+
</profile>
192+
</profiles>
193+
194+
</project>

‎kite-data/kite-data-oozie/src/main/java/org/kitesdk/data/oozie/KiteLauncherURIHandler.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/**
2+
* Copyright 2015 Cloudera Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.kitesdk.data.oozie;
17+
18+
import java.net.URI;
19+
import java.util.Collections;
20+
import java.util.List;
21+
import org.apache.hadoop.conf.Configuration;
22+
import org.apache.oozie.action.hadoop.LauncherException;
23+
import org.apache.oozie.action.hadoop.LauncherURIHandler;
24+
25+
public class KiteLauncherURIHandler implements LauncherURIHandler {
26+
27+
@Override
28+
public boolean create(final URI uri, final Configuration conf)
29+
throws LauncherException {
30+
throw new UnsupportedOperationException(
31+
"Creation of resources is not supported for Kite URIs.");
32+
}
33+
34+
@Override
35+
public boolean delete(final URI uri, final Configuration conf)
36+
throws LauncherException {
37+
throw new UnsupportedOperationException(
38+
"Deletion of resources is not supported for Kite URIs.");
39+
}
40+
41+
@Override
42+
public List<Class<?>> getClassesForLauncher() {
43+
return Collections.emptyList();
44+
}
45+
46+
}

‎kite-data/kite-data-oozie/src/main/java/org/kitesdk/data/oozie/KiteURIHandler.java‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
/**
2+
* Copyright 2015 Cloudera Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.kitesdk.data.oozie;
17+
18+
import java.net.URI;
19+
import java.util.Arrays;
20+
import java.util.HashSet;
21+
import java.util.List;
22+
import java.util.Set;
23+
import org.apache.avro.generic.GenericRecord;
24+
import org.apache.hadoop.conf.Configuration;
25+
import org.apache.oozie.ErrorCode;
26+
import org.apache.oozie.XException;
27+
import org.apache.oozie.action.hadoop.LauncherURIHandler;
28+
import org.apache.oozie.dependency.URIHandler;
29+
import org.apache.oozie.dependency.URIHandlerException;
30+
import org.apache.oozie.service.HadoopAccessorException;
31+
import org.apache.oozie.service.URIHandlerService;
32+
import org.apache.oozie.util.XLog;
33+
import org.kitesdk.data.DatasetException;
34+
import org.kitesdk.data.DatasetNotFoundException;
35+
import org.kitesdk.data.Datasets;
36+
import org.kitesdk.data.Signalable;
37+
import org.kitesdk.data.URIBuilder;
38+
import org.kitesdk.data.View;
39+
40+
/**
41+
* A Kite URI handler that works with {@link Signalable} views.
42+
*
43+
* To be considered as {@link #exists(URI, Configuration, String) existing} the
44+
* view must have been signaled as ready.
45+
*/
46+
public class KiteURIHandler implements URIHandler {
47+
48+
private static final XLog LOG = XLog.getLog(KiteURIHandler.class);
49+
50+
private Set<String> supportedSchemes;
51+
private List<Class<?>> classesToShip;
52+
53+
@Override
54+
public void init(final Configuration conf) {
55+
supportedSchemes = new HashSet<String>();
56+
final String[] schemes = conf.getStrings(
57+
URIHandlerService.URI_HANDLER_SUPPORTED_SCHEMES_PREFIX
58+
+ this.getClass().getSimpleName()
59+
+ URIHandlerService.URI_HANDLER_SUPPORTED_SCHEMES_SUFFIX, "view", "dataset");
60+
supportedSchemes.addAll(Arrays.asList(schemes));
61+
classesToShip = new KiteLauncherURIHandler().getClassesForLauncher();
62+
}
63+
64+
@Override
65+
public Set<String> getSupportedSchemes() {
66+
return supportedSchemes;
67+
}
68+
69+
@Override
70+
public Class<? extends LauncherURIHandler> getLauncherURIHandlerClass() {
71+
return KiteLauncherURIHandler.class;
72+
}
73+
74+
@Override
75+
public List<Class<?>> getClassesForLauncher() {
76+
return classesToShip;
77+
}
78+
79+
@Override
80+
public DependencyType getDependencyType(final URI uri)
81+
throws URIHandlerException {
82+
return DependencyType.PULL;
83+
}
84+
85+
@Override
86+
public void registerForNotification(final URI uri, final Configuration conf,
87+
final String user, final String actionID) throws URIHandlerException {
88+
throw new UnsupportedOperationException(
89+
"Notifications are not supported for " + uri);
90+
}
91+
92+
@Override
93+
public boolean unregisterFromNotification(final URI uri, final String actionID) {
94+
throw new UnsupportedOperationException(
95+
"Notifications are not supported for " + uri);
96+
}
97+
98+
@Override
99+
public Context getContext(final URI uri, final Configuration conf,
100+
final String user) throws URIHandlerException {
101+
return null;
102+
}
103+
104+
@SuppressWarnings("rawtypes")
105+
@Override
106+
public boolean exists(final URI uri, final Context context) throws URIHandlerException {
107+
try {
108+
View<GenericRecord> view = Datasets.load(uri);
109+
if(view instanceof Signalable) {
110+
return ((Signalable)view).isReady();
111+
}
112+
} catch (IllegalArgumentException ex) {
113+
LOG.error("the URI " + uri + " was not a view or dataset");
114+
} catch (DatasetNotFoundException ex) {
115+
LOG.error("the dataset for the URI "+ uri +" did not actually exist");
116+
} catch (DatasetException e) {
117+
throw new HadoopAccessorException(ErrorCode.E0902, e);
118+
}
119+
// didn't meet all the requirements for a URI/view that we want to use with oozie
120+
return false;
121+
}
122+
123+
@Override
124+
public boolean exists(final URI uri, final Configuration conf, final String user)
125+
throws URIHandlerException {
126+
// currently does not handle access limitations between the oozie user and
127+
// a potential dataset owner
128+
return exists(uri, null);
129+
}
130+
131+
@Override
132+
public String getURIWithDoneFlag(final String uri, final String doneFlag)
133+
throws URIHandlerException {
134+
return uri;
135+
}
136+
137+
@Override
138+
public void validate(final String uri) throws URIHandlerException {
139+
try {
140+
URI uriParsed = URI.create(uri);
141+
String scheme = uriParsed.getScheme();
142+
if(!(URIBuilder.VIEW_SCHEME.equals(scheme) || URIBuilder.DATASET_SCHEME.equals(scheme))) {
143+
LOG.error("Unexpected scheme: view, uri was "+ uri);
144+
XException xException = new XException(ErrorCode.E0904, scheme, uri);
145+
throw new URIHandlerException(xException);
146+
}
147+
} catch (IllegalArgumentException iae) {
148+
XException xException = new XException(ErrorCode.E0906, iae);
149+
throw new URIHandlerException(xException);
150+
}
151+
}
152+
153+
@Override
154+
public void destroy() {
155+
}
156+
157+
}

0 commit comments

Comments
 (0)