Skip to content

Commit

Permalink
Merge pull request #4646 from nadment/4645
Browse files Browse the repository at this point in the history 
Move ValueMetaJson to Core library #4645
  • Loading branch information
@hansva
hansva authored Dec 10, 2024
2 parents 689528c + e0fde51 commit e9371b2
Show file tree
Hide file tree
Showing 21 changed files with 354 additions and 533 deletions.
14 changes: 14 additions & 0 deletions core/src/main/java/org/apache/hop/core/row/IValueMeta.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.hop.core.row;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import java.io.DataInputStream;
import java.io.DataOutputStream;
Expand Down Expand Up @@ -111,6 +112,12 @@
* <td>java.lang.byte[</td>
* <td>An array of bytes that contain any type of binary data.</td>
* </tr>
* <tr>
* <td>Json</td>
* <td>TYPE_JSON</td>
* <td>com.fasterxml.jackson.databind.JsonNode</td>
* <td>A native Json object.</td>
* </tr>
* </Table>
*
* <p><b>Storage Types</b>
Expand Down Expand Up @@ -185,6 +192,9 @@ public interface IValueMeta extends Cloneable {
/** Value type indicating that the value contains an Internet address */
int TYPE_INET = 10;

/** Value type indicating that the value contains a native JSON object */
int TYPE_JSON = 11;

/** Value type indicating that the value contains an Avro Record */
int TYPE_AVRO = 20;

Expand All @@ -202,6 +212,7 @@ public interface IValueMeta extends Cloneable {
"Binary",
"Timestamp",
"Internet Address",
"Json"
};

/** The storage type is the same as the indicated value type */
Expand Down Expand Up @@ -893,6 +904,9 @@ static String getTypeDescription(int type) {
/** Convert the supplied data to a Boolean */
Boolean getBoolean(Object object) throws HopValueException;

/** Convert the supplied data to a Json */
JsonNode getJson(Object object) throws HopValueException;

/** Convert the supplied data to binary data */
byte[] getBinary(Object object) throws HopValueException;

Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
name = "Avro Record",
description = "This type wraps around an Avro Record",
image = "images/avro.svg")
public class ValueMetaAvroRecord extends ValueMetaBase implements IValueMeta {
public class ValueMetaAvroRecord extends ValueMetaBase {

private Schema schema;
private static final String CONST_SCHEMA = "schema";
Expand Down
216 changes: 214 additions & 2 deletions core/src/main/java/org/apache/hop/core/row/value/ValueMetaBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,14 @@
package org.apache.hop.core.row.value;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.core.json.JsonReadFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.databind.node.BooleanNode;
import com.fasterxml.jackson.databind.node.DecimalNode;
import com.fasterxml.jackson.databind.node.DoubleNode;
import com.fasterxml.jackson.databind.node.LongNode;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
Expand Down Expand Up @@ -58,6 +66,7 @@
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopFileException;
import org.apache.hop.core.exception.HopValueException;
import org.apache.hop.core.json.HopJson;
import org.apache.hop.core.logging.HopLogStore;
import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.row.IValueMeta;
Expand Down Expand Up @@ -1201,6 +1210,25 @@ protected synchronized Double convertStringToNumber(String string) throws HopVal
}
}

public String convertJsonToString(JsonNode jsonNode) throws HopValueException {
try {
ObjectMapper objectMapper = HopJson.newMapper();
return objectMapper.writeValueAsString(jsonNode);
} catch (Exception e) {
throw new HopValueException("Error converting JSON value to String", e);
}
}

public JsonNode convertStringToJson(String jsonString) throws HopValueException {
try {
ObjectMapper objectMapper =
JsonMapper.builder().enable(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES).build();
return objectMapper.readTree(jsonString);
} catch (Exception e) {
throw new HopValueException("Error converting string to JSON value: '" + jsonString + "'", e);
}
}

@Override
public synchronized SimpleDateFormat getDateFormat() {
return getDateFormat(getType());
Expand Down Expand Up @@ -2197,6 +2225,127 @@ public String getString(Object object) throws HopValueException {
}
}

@Override
public JsonNode getJson(Object object) throws HopValueException {
if (object == null) {
return null;
}

switch (type) {
case TYPE_JSON:
switch (storageType) {
case STORAGE_TYPE_NORMAL:
return (JsonNode) object;
case STORAGE_TYPE_BINARY_STRING:
return convertStringToJson(convertBinaryStringToString((byte[]) object));
case STORAGE_TYPE_INDEXED:
return (JsonNode) index[((Integer) object)];
default:
throw new HopValueException(
toString() + MSG_UNKNOWN_STORAGE_TYPE + storageType + MSG_SPECIFIED);
}
case TYPE_STRING:
switch (storageType) {
case STORAGE_TYPE_NORMAL:
return convertStringToJson((String) object);
case STORAGE_TYPE_BINARY_STRING:
return convertStringToJson((String) convertBinaryStringToNativeType((byte[]) object));
case STORAGE_TYPE_INDEXED:
return convertStringToJson((String) index[(Integer) object]);
default:
throw new HopValueException(
toString() + MSG_UNKNOWN_STORAGE_TYPE + storageType + MSG_SPECIFIED);
}
case TYPE_NUMBER:
Double number;
switch (storageType) {
case STORAGE_TYPE_NORMAL:
number = (Double) object;
break;
case STORAGE_TYPE_BINARY_STRING:
number = convertStringToNumber(convertBinaryStringToString((byte[]) object));
break;
case STORAGE_TYPE_INDEXED:
number = (Double) index[(Integer) object];
break;
default:
throw new HopValueException(
toString() + MSG_UNKNOWN_STORAGE_TYPE + storageType + MSG_SPECIFIED);
}
return new DoubleNode(number);

case TYPE_INTEGER:
Long integer;
switch (storageType) {
case STORAGE_TYPE_NORMAL:
integer = (Long) object;
break;
case STORAGE_TYPE_BINARY_STRING:
integer = (Long) convertBinaryStringToNativeType((byte[]) object);
break;
case STORAGE_TYPE_INDEXED:
integer = (Long) index[(Integer) object];
break;
default:
throw new HopValueException(
toString() + MSG_UNKNOWN_STORAGE_TYPE + storageType + MSG_SPECIFIED);
}
return new LongNode(integer);

case TYPE_BIGNUMBER:
BigDecimal bigDecimal;
switch (storageType) {
case STORAGE_TYPE_NORMAL:
bigDecimal = (BigDecimal) object;
break;
case STORAGE_TYPE_BINARY_STRING:
bigDecimal = (BigDecimal) convertBinaryStringToNativeType((byte[]) object);
break;
case STORAGE_TYPE_INDEXED:
bigDecimal = (BigDecimal) index[(Integer) object];
break;
default:
throw new HopValueException(
toString() + MSG_UNKNOWN_STORAGE_TYPE + storageType + MSG_SPECIFIED);
}
return new DecimalNode(bigDecimal);

case TYPE_BOOLEAN:
boolean bool;
switch (storageType) {
case STORAGE_TYPE_NORMAL:
bool = (Boolean) object;
break;
case STORAGE_TYPE_BINARY_STRING:
bool = (Boolean) convertBinaryStringToNativeType((byte[]) object);
break;
case STORAGE_TYPE_INDEXED:
bool = (Boolean) index[(Integer) object];
break;
default:
throw new HopValueException(
toString() + MSG_UNKNOWN_STORAGE_TYPE + storageType + MSG_SPECIFIED);
}
return BooleanNode.valueOf(bool);

case TYPE_DATE:
throw new HopValueException(
toString() + " : I don't know how to convert a date to a JSON object.");
case TYPE_TIMESTAMP:
throw new HopValueException(
toString() + " : I don't know how to convert a timestamp to a JSON object.");
case TYPE_BINARY:
throw new HopValueException(
toString() + " : I don't know how to convert a binary value to JSON object.");
case TYPE_SERIALIZABLE:
throw new HopValueException(
toString() + " : I don't know how to convert a serializable value to JSON object.");

default:
throw new HopValueException(toString() + " : Unknown type " + type + MSG_SPECIFIED);
}
}

protected String trim(String string) {
switch (getTrimType()) {
case TRIM_TYPE_NONE:
Expand Down Expand Up @@ -2928,6 +3077,21 @@ public byte[] getBinaryString(Object object) throws HopValueException {
+ MSG_SPECIFIED);
}

case TYPE_JSON:
switch (storageType) {
case STORAGE_TYPE_NORMAL:
return convertStringToBinaryString(getString(object));
case STORAGE_TYPE_BINARY_STRING:
return convertStringToBinaryString(
getString(convertStringToJson(convertBinaryStringToString((byte[]) object))));
case STORAGE_TYPE_INDEXED:
return convertStringToBinaryString(
convertJsonToString((JsonNode) index[((Integer) object)]));
default:
throw new HopValueException(
toString() + MSG_UNKNOWN_STORAGE_TYPE + storageType + MSG_SPECIFIED);
}

case TYPE_SERIALIZABLE:
switch (storageType) {
case STORAGE_TYPE_NORMAL:
Expand Down Expand Up @@ -3191,6 +3355,9 @@ public void writeData(DataOutputStream outputStream, Object object) throws HopFi
case TYPE_BINARY:
writeBinary(outputStream, (byte[]) object);
break;
case TYPE_JSON:
writeJson(outputStream, (JsonNode) object);
break;
case TYPE_INET:
writeBinary(outputStream, ((InetAddress) object).getAddress());
break;
Expand Down Expand Up @@ -3259,6 +3426,8 @@ public Object readData(DataInputStream inputStream)
return readBoolean(inputStream);
case TYPE_BINARY:
return readBinary(inputStream);
case TYPE_JSON:
return readJson(inputStream);
case TYPE_INET:
return InetAddress.getByAddress(readBinary(inputStream));
default:
Expand Down Expand Up @@ -3296,6 +3465,19 @@ protected void writeString(DataOutputStream outputStream, String string) throws
}
}

protected void writeJson(DataOutputStream outputStream, JsonNode jsonNode) throws IOException {
// Write the length and then the bytes
if (jsonNode == null) {
outputStream.writeInt(-1);
} else {
ObjectMapper objectMapper = new ObjectMapper();
String string = objectMapper.writeValueAsString(jsonNode);
byte[] chars = string.getBytes(Const.XML_ENCODING);
outputStream.writeInt(chars.length);
outputStream.write(chars);
}
}

protected void writeBinaryString(DataOutputStream outputStream, byte[] binaryString)
throws IOException {
// Write the length and then the bytes
Expand All @@ -3320,7 +3502,7 @@ protected String readString(DataInputStream inputStream) throws IOException {
return new String(chars, Const.XML_ENCODING);
}

protected byte[] readBinaryString(DataInputStream inputStream) throws IOException {
protected JsonNode readJson(DataInputStream inputStream) throws IOException {
// Read the inputLength and then the bytes
int inputLength = inputStream.readInt();
if (inputLength < 0) {
Expand All @@ -3330,6 +3512,20 @@ protected byte[] readBinaryString(DataInputStream inputStream) throws IOExceptio
byte[] chars = new byte[inputLength];
inputStream.readFully(chars);

ObjectMapper objectMapper = new ObjectMapper();
return objectMapper.readTree(chars, 0, inputLength);
}

protected byte[] readBinaryString(DataInputStream inputStream) throws IOException {
// Read the length and then the bytes
int inputLength = inputStream.readInt();
if (inputLength < 0) {
return null;
}

byte[] chars = new byte[inputLength];
inputStream.readFully(chars);

return chars;
}

Expand Down Expand Up @@ -3440,6 +3636,9 @@ public void writeMeta(DataOutputStream outputStream) throws HopFileException {
case TYPE_BINARY:
writeBinary(outputStream, (byte[]) index[i]);
break;
case TYPE_JSON:
writeJson(outputStream, (JsonNode) index[i]);
break;
default:
throw new HopFileException(
this
Expand Down Expand Up @@ -3578,6 +3777,9 @@ public void readMetaData(DataInputStream inputStream) throws HopFileException {
case TYPE_BINARY:
index[i] = readBinary(inputStream);
break;
case TYPE_JSON:
index[i] = readJson(inputStream);
break;
default:
throw new HopFileException(
this
Expand Down Expand Up @@ -4291,6 +4493,8 @@ public Object convertData(IValueMeta meta2, Object data2) throws HopValueExcepti
return meta2.getBoolean(data2);
case TYPE_BINARY:
return meta2.getBinary(data2);
case TYPE_JSON:
return meta2.getJson(data2);
default:
throw new HopValueException(this + CONST_CANNOT_CONVERT + getType());
}
Expand Down Expand Up @@ -4322,6 +4526,8 @@ public Object convertDataCompatible(IValueMeta meta2, Object data2) throws HopVa
return meta2.getBoolean(data2);
case TYPE_BINARY:
return meta2.getBinary(data2);
case TYPE_JSON:
return meta2.getJson(data2);
default:
throw new HopValueException(this + CONST_CANNOT_CONVERT + getType());
}
Expand Down Expand Up @@ -4349,7 +4555,6 @@ public Object convertDataUsingConversionMetaData(Object data) throws HopValueExc
// storageMetaData to get the correct conversion mask
// That way we're always sure that a conversion works both ways.
//

switch (conversionMetadata.getType()) {
case TYPE_STRING:
return getString(data);
Expand All @@ -4367,6 +4572,8 @@ public Object convertDataUsingConversionMetaData(Object data) throws HopValueExc
return getBinary(data);
case TYPE_TIMESTAMP:
return getDate(data);
case TYPE_JSON:
return getJson(data);
default:
throw new HopValueException(this + CONST_CANNOT_CONVERT + conversionMetadata.getType());
}
Expand Down Expand Up @@ -4557,6 +4764,8 @@ public int hashCode(Object object) throws HopValueException {
case TYPE_INET:
hash ^= 256;
break;
case TYPE_JSON:
hash ^= 512;
case TYPE_NONE:
break;
default:
Expand Down Expand Up @@ -4591,6 +4800,9 @@ public int hashCode(Object object) throws HopValueException {
case TYPE_INET:
hash ^= object.hashCode();
break;
case TYPE_JSON:
hash ^= object.hashCode();
break;
case TYPE_NONE:
break;
default:
Expand Down
Loading

0 comments on commit e9371b2

Please sign in to comment.