Skip to content

Commit

Permalink
parsing NULLs for double columns is fixed
Browse files Browse the repository at this point in the history
  • Loading branch information
fabuzaid21 committed Feb 8, 2018
1 parent 9d70a74 commit 8f5fdff
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 19 deletions.
11 changes: 0 additions & 11 deletions sql/load.sql
Original file line number Diff line number Diff line change
@@ -1,14 +1,3 @@
IMPORT FROM CSV FILE 'core/demo/sample.csv' INTO sample(usage double, latency
double, location string, version string);

IMPORT FROM CSV FILE 'core/demo/mobile_data.csv' INTO mobile_data(record_id
string, user_id string, state string, hw_make string, hw_model string,
firmware_version string, app_version string, avg_temp double, battery_drain
double, trip_time double);

IMPORT FROM CSV FILE 'wikiticker.csv' INTO wiki(time string, user string, page
string, channel string, namespace string, comment string, metroCode string,
cityName string, regionName string, regionIsoCode string, countryName string,
countryIsoCode string, isAnonymous string, isMinor string, isNew string,
isRobot string, isUnpatrolled string, delta double, added double, deleted
double);
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package edu.stanford.futuredata.macrobase.sql;

import static com.google.common.collect.ImmutableList.toImmutableList;
import static java.util.stream.Collectors.toList;

import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -232,7 +231,7 @@ private List<SingleColumn> removeUDFsInSelect(List<SelectItem> selectItems) {
final SingleColumn col = (SingleColumn) item;
if (col.getExpression() instanceof FunctionCall) {
functionCalls.add(col);
it.remove();
it.remove();
}
}
}
Expand Down Expand Up @@ -341,14 +340,15 @@ private DataFrame getTable(String tableName) throws MacrobaseSQLException {
}

/**
* Evaluate only the UDFs of SQL query and return a Map of column names -> double arrays.
* If there are no UDFs (i.e. @param udfCols is empty), the Map is empty.
* Evaluate only the UDFs of SQL query and return a Map of column names -> double arrays. If
* there are no UDFs (i.e. @param udfCols is empty), an empty Map is returned.
*
* @param inputDf The DataFrame to evaluate the UDFs on
* @param udfCols The List of UDFs to evaluate
* @return The Map of new columns to be added
*/
private Map<String, double[]> evaluateUDFs(final DataFrame inputDf, final List<SingleColumn> udfCols)
private Map<String, double[]> evaluateUDFs(final DataFrame inputDf,
final List<SingleColumn> udfCols)
throws MacrobaseException {
final Map<String, double[]> newColumns = new HashMap<>();
for (SingleColumn udfCol : udfCols) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collector;

public class SingleColumn extends SelectItem {

Expand Down Expand Up @@ -87,8 +86,8 @@ public String toString() {
}

/**
* @return If the Expression is a Function Call (e.g., a UDF), concatenate the function name
* and the arguments with "_". Otherwise, return the output of toString()
* @return If the Expression is a Function Call (e.g., a UDF), concatenate the function name and
* the arguments with "_". Otherwise, return the output of toString()
*/
private String formatForCol(final Expression expr) {
if (expr instanceof FunctionCall) {
Expand Down

0 comments on commit 8f5fdff

Please sign in to comment.