diff --git a/README.md b/README.md index acf4096..7d60a2f 100755 --- a/README.md +++ b/README.md @@ -158,6 +158,17 @@ To list available columns, run `!columns` command: Next, try SQL queries over data +For example: +```shell +!connect jdbc:calcite:model=src/main/resources/model.json admin admin +``` +```shell +select "open" from NIFTY_50; +``` +```shell +SELECT * FROM "Sectoral"."NIFTY_ENERGY" INNER JOIN "NIFTY_50" ON "Sectoral"."NIFTY_ENERGY"."symbol" = "NIFTY_50"."symbol"; +``` + ### Other Details * Log File: application.log diff --git a/pom.xml b/pom.xml index 0fa9ffd..ac22d2c 100644 --- a/pom.xml +++ b/pom.xml @@ -14,8 +14,8 @@ maven-compiler-plugin 3.8.1 - 7 - 7 + 8 + 8 @@ -85,10 +85,20 @@ ${maven.build.timestamp} 1.32.0 - 1.15.0 + 1.22.0 + + org.htmlunit + htmlunit + 3.1.0 + + + org.apache.commons + commons-lang3 + 3.12.0 + org.apache.calcite.avatica avatica @@ -99,15 +109,10 @@ calcite-core ${calcite.version} - - com.google.http-client - google-http-client - 1.30.2 - com.google.code.gson gson - 2.8.9 + 2.8.5 sqlline @@ -125,5 +130,11 @@ slf4j-log4j12 1.7.25 + + junit + junit + 4.13.2 + test + - \ No newline at end of file + diff --git a/src/main/java/org/gitcloned/calcite/adapter/nse/NSESchema.java b/src/main/java/org/gitcloned/calcite/adapter/nse/NSESchema.java index 4b1d1bb..64464e3 100755 --- a/src/main/java/org/gitcloned/calcite/adapter/nse/NSESchema.java +++ b/src/main/java/org/gitcloned/calcite/adapter/nse/NSESchema.java @@ -1,6 +1,10 @@ package org.gitcloned.calcite.adapter.nse; import com.google.common.collect.ImmutableMap; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.calcite.schema.Table; import org.apache.calcite.schema.impl.AbstractSchema; import org.gitcloned.nse.NseData; @@ -8,6 +12,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.sql.Connection; import java.util.Locale; import java.util.Map; @@ -23,13 +29,13 @@ public class NSESchema extends AbstractSchema { private Map tableMap; private final NseData nse; + private final String MASTER_SCHEMA_URL = "https://www.nseindia.com/api/equity-master"; public NSESchema(String group) { this.group = group; - NseHTTP nseHttp = new NseHTTP("https://www1.nseindia.com/live_market/dynaContent/live_watch/stock_watch/"); - - this.nse = new NseData(nseHttp); + NseHTTP nseHttp = new NseHTTP("https://www.nseindia.com", group); + this.nse = new NseData("https://www.nseindia.com/api/equity-stockIndices?index=", nseHttp); } public NseData getNseSession() { @@ -58,67 +64,38 @@ private Map createTableMap() { logger.debug(String.format("getting tables for group: '%s'", this.group)); /** - * List all tables under different categories in NSE + * List all tables under different categories in NSE from https://www.nseindia.com/api/equity-master */ - if (this.group.equals("Broad Market Indices")) { - - NSEScannableTable table = (NSEScannableTable) createTable("nifty", "Nifty50"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("juniorNifty", "Nifty Junior"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("niftyMidcap50", "Nifty Midcap 50"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("niftyMidcap150Online", "Nifty Midcap 150"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("niftySmallcap50Online", "Nifty Smlcap 50"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("niftySmallcap250Online", "Nifty Smlcap 250"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - } else if (this.group.equals("Sectoral Indices")) { - - NSEScannableTable table = (NSEScannableTable) createTable("cnxAuto", "Nifty Auto"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("bankNifty", "Nifty Bank"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("cnxEnergy", "Nifty Energy"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("cnxFinance", "Nifty Financial Services"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("cnxFMCG", "Nifty FMCG"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("cnxit", "Nifty IT"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("cnxMetal", "Nifty Metal"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("cnxPharma", "Nifty Pharma"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - - table = (NSEScannableTable) createTable("cnxRealty", "Nifty Realty"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - } else if (this.group.equals("Thematic Indices")) { - - NSEScannableTable table = (NSEScannableTable) createTable("cnxCommodities", "Nifty Commodities"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - } else if (this.group.equals("Strategy Indices")) { - - NSEScannableTable table = (NSEScannableTable) createTable("cnxDividendOppt", "Nifty Dividend Opportunities 50"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); - } else { - - NSEScannableTable table = (NSEScannableTable) createTable("sovGold", "Sovereign Gold Bonds"); - builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); + try { + String masterJson = this.nse.getRequestBuilder().sendGetRequest(MASTER_SCHEMA_URL).getContentAsString(); + + JsonParser parser = new JsonParser(); + JsonElement tree = parser.parse(masterJson); + + if (tree.isJsonObject()) { + + for(String groupName : ((JsonObject)tree).keySet()) { + if(this.group.equals(groupName)) { + logger.info(String.format("Loading group '%s'", groupName)); + JsonElement groupData = ((JsonObject)tree).get(groupName); + if (groupData.isJsonArray()) { + JsonArray group = groupData.getAsJsonArray(); + for (JsonElement tableData : group) { + if (tableData.isJsonPrimitive()) { + String tableName = tableData.getAsString(); + logger.info(String.format("Loading group '%s' table '%s'", groupName, tableName)); + + NSEScannableTable table = (NSEScannableTable) createTable(tableName, tableName.replace(" ", "_")); + builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table); + } + } + } + } + } + } + } + catch (IOException e) { + throw new RuntimeException(e); } return builder.build(); diff --git a/src/main/java/org/gitcloned/calcite/adapter/nse/NSESchemaFactory.java b/src/main/java/org/gitcloned/calcite/adapter/nse/NSESchemaFactory.java index 3fe7ddd..ecc8548 100755 --- a/src/main/java/org/gitcloned/calcite/adapter/nse/NSESchemaFactory.java +++ b/src/main/java/org/gitcloned/calcite/adapter/nse/NSESchemaFactory.java @@ -9,8 +9,6 @@ public class NSESchemaFactory implements SchemaFactory { public Schema create(SchemaPlus schemaPlus, String s, Map map) { - - NSESchema schema = new NSESchema((String)map.get("group")); - return schema; + return new NSESchema((String)map.get("group")); } } diff --git a/src/main/java/org/gitcloned/nse/NseData.java b/src/main/java/org/gitcloned/nse/NseData.java index d5ef3b6..ab435e6 100755 --- a/src/main/java/org/gitcloned/nse/NseData.java +++ b/src/main/java/org/gitcloned/nse/NseData.java @@ -1,24 +1,27 @@ package org.gitcloned.nse; -import com.google.api.client.http.HttpRequest; -import com.google.api.client.http.HttpResponse; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.gitcloned.nse.http.NseHTTP; +import org.htmlunit.WebResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; public class NseData { Logger logger = LoggerFactory.getLogger(NseData.class); private final NseHTTP requestBuilder; + public String EQUITY_URL; - public NseData(NseHTTP requestBuilder) { + public NseData(String EQUITY_URL, NseHTTP requestBuilder) { + this.EQUITY_URL = EQUITY_URL; this.requestBuilder = requestBuilder; } @@ -26,28 +29,14 @@ public NseHTTP getRequestBuilder() { return requestBuilder; } - private boolean isSuccessfulResponse (int statusCode) { - - if (statusCode - 200 < 0) return false; - if (statusCode - 200 >= 100) return false; - return true; - } - public JsonArray scanTable (String groupName, String tableName) throws IOException, RuntimeException { + WebResponse response = this.requestBuilder.sendGetRequest(EQUITY_URL + URLEncoder.encode(tableName, StandardCharsets.UTF_8.toString())); - StringBuilder api = new StringBuilder(""); - api.append(tableName); - api.append("StockWatch.json"); - - HttpRequest request = this.requestBuilder.buildGetRequest(api.toString(), null); - - HttpResponse response = request.execute(); - - if (!isSuccessfulResponse(response.getStatusCode())) { - throw new RuntimeException("Data Request Failed, status code (" + response.getStatusCode() + "), Response: " + response.parseAsString()); + if (!NseHTTP.isSuccessfulResponse(response.getStatusCode())) { + throw new RuntimeException("Data Request Failed, status code (" + response.getStatusCode() + "), Response: " + response.getStatusMessage()); } - String responseString = response.parseAsString(); + String responseString = response.getContentAsString(); JsonParser parser = new JsonParser(); JsonElement tree = parser.parse(responseString); @@ -55,7 +44,7 @@ public JsonArray scanTable (String groupName, String tableName) throws IOExcepti if (tree.isJsonObject()) { JsonElement rows = tree.getAsJsonObject().get("data"); - JsonElement time = tree.getAsJsonObject().get("time"); + JsonElement time = tree.getAsJsonObject().get("timestamp"); if (time.isJsonNull()) { throw new RuntimeException("Data Request Failed: Response is not a valid json, cannot find the 'time' of the data response"); diff --git a/src/main/java/org/gitcloned/nse/http/NseHTTP.java b/src/main/java/org/gitcloned/nse/http/NseHTTP.java index 9b5c4a1..9cf029c 100755 --- a/src/main/java/org/gitcloned/nse/http/NseHTTP.java +++ b/src/main/java/org/gitcloned/nse/http/NseHTTP.java @@ -1,64 +1,45 @@ package org.gitcloned.nse.http; -import com.google.api.client.http.*; -import com.google.api.client.http.javanet.NetHttpTransport; +import org.apache.commons.lang3.StringUtils; +import org.htmlunit.BrowserVersion; +import org.htmlunit.WebClient; +import org.htmlunit.WebResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.security.GeneralSecurityException; import java.util.Map; public class NseHTTP { Logger logger = LoggerFactory.getLogger(NseHTTP.class); - private final String BASE_URL; - private HttpRequestFactory requestFactory; - - public NseHTTP(String BASE_URL) { - - this.BASE_URL = BASE_URL; - this.requestFactory = new NetHttpTransport().createRequestFactory(); - - try { - - NetHttpTransport.Builder builder = new NetHttpTransport.Builder(); - builder.doNotValidateCertificate(); - - this.requestFactory = builder.build().createRequestFactory(); - } catch (GeneralSecurityException e) { - throw new RuntimeException("Got General Security exception: " + e.getMessage()); - } - } - - private void setHeaders (HttpRequest request, Map headers) { - if (headers != null) { - - HttpHeaders httpHeaders = request.getHeaders(); - - for (String key : headers.keySet()) { - httpHeaders.set(key, headers.get(key)); + private WebClient webClient; + + public NseHTTP(String BASE_URL_FOR_COOKIES, String target) { + this.webClient = new WebClient(BrowserVersion.CHROME); + webClient.getOptions().setJavaScriptEnabled(false); + webClient.getOptions().setCssEnabled(false); + if(StringUtils.isNotBlank(BASE_URL_FOR_COOKIES)) { + logger.info(String.format("Initializing web client cookies from %s for %s", BASE_URL_FOR_COOKIES, target)); + try { + WebResponse response = webClient.getPage(BASE_URL_FOR_COOKIES).getWebResponse(); + if(200 != response.getStatusCode()) { + throw new RuntimeException(String.format("Can not initialize cookies using %s with result %s, code %s", BASE_URL_FOR_COOKIES, response.getStatusMessage(), response.getStatusCode())); + } + } + catch (IOException e) { + throw new RuntimeException(e); } } } - - private void setAuth (HttpRequest request) { - - // No auth required + public static boolean isSuccessfulResponse (int statusCode) { + if (statusCode - 200 < 0) return false; + if (statusCode - 200 >= 100) return false; + return true; } - - public HttpRequest buildGetRequest (String api, Map headers) throws IOException { - - logger.info(String.format("Hitting api '%s'", BASE_URL + api)); - - HttpRequest request = requestFactory.buildGetRequest( - new GenericUrl(BASE_URL + api)); - - setHeaders(request, headers); - - setAuth(request); - - return request; + public WebResponse sendGetRequest (String api) throws IOException { + logger.info(String.format("Hitting api '%s'", api)); + return webClient.getPage(api).getWebResponse(); } } diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index b0029b2..378d523 100755 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -2,11 +2,11 @@ log4j.rootLogger=DEBUG, consoleAppender, fileAppender log4j.appender.consoleAppender=org.apache.log4j.ConsoleAppender log4j.appender.consoleAppender.layout=org.apache.log4j.PatternLayout -log4j.appender.consoleAppender.layout.ConversionPattern=[%t] %-5p %c %x - %m% +log4j.appender.consoleAppender.layout.ConversionPattern=[%t] %-5p %c %x - %m%n log4j.appender.consoleAppender.Threshold=INFO log4j.appender.fileAppender=org.apache.log4j.RollingFileAppender log4j.appender.fileAppender.layout=org.apache.log4j.PatternLayout log4j.appender.fileAppender.layout.ConversionPattern=[%t] %-5p %c %x - %m%n log4j.appender.fileAppender.File=application.log -log4j.appender.fileAppender.Threshold=DEBUG \ No newline at end of file +log4j.appender.fileAppender.Threshold=DEBUG diff --git a/src/main/resources/model.json b/src/main/resources/model.json index 592ea89..892ef39 100755 --- a/src/main/resources/model.json +++ b/src/main/resources/model.json @@ -61,4 +61,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/test/java/RestSQL.java b/src/test/java/RestSQL.java new file mode 100644 index 0000000..f4b1e3d --- /dev/null +++ b/src/test/java/RestSQL.java @@ -0,0 +1,72 @@ +import org.junit.Test; +import sqlline.BuiltInProperty; +import sqlline.DispatchCallback; +import sqlline.SqlLine; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; + +public class RestSQL { + + public static void printResultSet(ResultSet rs) throws SQLException { + ResultSetMetaData metaData = rs.getMetaData(); + int columnCount = metaData.getColumnCount(); + + // Print column names + for (int i = 1; i <= columnCount; i++) { + System.out.print(metaData.getColumnName(i) + "\t"); + } + System.out.println(); + + // Print each row of data + while (rs.next()) { + for (int i = 1; i <= columnCount; i++) { + System.out.print(rs.getString(i) + "\t"); + } + System.out.println(); + } + } + + @Test + public void jdbc() throws Exception { + SqlLine sqlLine = new SqlLine(); + sqlLine.getOpts().set(BuiltInProperty.ISOLATION, "TRANSACTION_NONE"); + sqlLine.getOpts().set(BuiltInProperty.MAX_WIDTH, 120); + + String url = "jdbc:calcite:model=src/main/resources/model.json"; + String user = "admin"; + String password = "admin"; + + Connection connection = DriverManager.getConnection(url, user, password); + + DatabaseMetaData metadata = connection.getMetaData(); + System.out.println("Database product name: " + metadata.getDatabaseProductName()); + System.out.println("Default transaction isolation: " + metadata.getDefaultTransactionIsolation()); + + Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery("select \"open\" from NIFTY_50"); + printResultSet(rs); + + ResultSet rs2 = stmt.executeQuery("SELECT * FROM \"Sectoral\".\"NIFTY_ENERGY\" INNER JOIN \"NIFTY_50\" ON \"Sectoral\".\"NIFTY_ENERGY\".\"symbol\" = \"NIFTY_50\".\"symbol\""); + printResultSet(rs2); + } + + @Test + public void sqlline() { + SqlLine sqlLine = new SqlLine(); + sqlLine.getOpts().set(BuiltInProperty.ISOLATION, "TRANSACTION_NONE"); + sqlLine.getOpts().set(BuiltInProperty.MAX_WIDTH, 120); + + DispatchCallback callback = new DispatchCallback(); + sqlLine.runCommands(Arrays.asList("!connect jdbc:calcite:model=src/main/resources/model.json admin admin", + "select \"open\" from NIFTY_50;", + "SELECT * FROM \"Sectoral\".\"NIFTY_ENERGY\" INNER JOIN \"NIFTY_50\" ON \"Sectoral\".\"NIFTY_ENERGY\".\"symbol\" = \"NIFTY_50\".\"symbol\"" + ), callback); + } +}