Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public final class AppConfig {
public static final String POSTGRESQL_USER = CONFIG.getString("POSTGRESQL_USER");
public static final String POSTGRESQL_PASSWORD = CONFIG.getString("POSTGRESQL_PASSWORD");
public static final String POSTGRESQL_NOTIFICATIONS_DB = CONFIG.getString("POSTGRESQL_NOTIFICATIONS_DB");
public static final String POSTGRESQL_CONFIGURATION_DB = CONFIG.getString("POSTGRESQL_CONFIGURATION_DB");
public static final String POSTGRESQL_AUDIT_DB = CONFIG.getString("POSTGRESQL_AUDIT_DB");
public static final String KAFKA_BOOTSTRAP_SERVERS = CONFIG.getString("KAFKA_BOOTSTRAP_SERVERS");
public static final String KAFKA_APPLICATION_ID = CONFIG.getString("KAFKA_APPLICATION_ID");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ public Behavior<Void> create() {
AppConfig.POSTGRESQL_PASSWORD,
AppConfig.POSTGRESQL_NOTIFICATIONS_DB,
AppConfig.POSTGRESQL_AUDIT_DB,
AppConfig.POSTGRESQL_CONFIGURATION_DB,
AppConfig.KAFKA_BOOTSTRAP_SERVERS,
"CLIENT_ID_API-" + UUID.randomUUID(),
AppConfig.SYSTEM_CONFIG_DIR,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ public final class AppConfig {
public static final String POSTGRESQL_USERS_DB = CONFIG.getString("POSTGRESQL_USERS_DB");
public static final String POSTGRESQL_NOTIFICATIONS_DB = CONFIG.getString("POSTGRESQL_NOTIFICATIONS_DB");
public static final String POSTGRESQL_AUDIT_DB = CONFIG.getString("POSTGRESQL_AUDIT_DB");
public static final String POSTGRESQL_CONFIGURATION_DB = CONFIG.getString("POSTGRESQL_CONFIGURATION_DB");
public static final String KAFKA_BOOTSTRAP_SERVERS = CONFIG.getString("KAFKA_BOOTSTRAP_SERVERS");
public static final String KAFKA_APPLICATION_ID = CONFIG.getString("KAFKA_APPLICATION_ID");
private static final String[] DGRAPH_ALPHA_HOSTS = CONFIG.getString("DGRAPH_HOSTS").split(",");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ public Behavior<Void> create() {
AppConfig.POSTGRESQL_PASSWORD,
AppConfig.POSTGRESQL_NOTIFICATIONS_DB,
AppConfig.POSTGRESQL_AUDIT_DB,
AppConfig.POSTGRESQL_CONFIGURATION_DB,
AppConfig.KAFKA_BOOTSTRAP_SERVERS,
"CLIENT_ID_API_KC-" + UUID.randomUUID(),
AppConfig.SYSTEM_CONFIG_DIR,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public final class AppConfig {
public static final String POSTGRESQL_USER = CONFIG.getString("POSTGRESQL_USER");
public static final String POSTGRESQL_PASSWORD = CONFIG.getString("POSTGRESQL_PASSWORD");
public static final String POSTGRESQL_NOTIFICATIONS_DB = CONFIG.getString("POSTGRESQL_NOTIFICATIONS_DB");
public static final String POSTGRESQL_CONFIGURATION_DB = CONFIG.getString("POSTGRESQL_CONFIGURATION_DB");
public static final String POSTGRESQL_AUDIT_DB = CONFIG.getString("POSTGRESQL_AUDIT_DB");
public static final String KAFKA_BOOTSTRAP_SERVERS = CONFIG.getString("KAFKA_BOOTSTRAP_SERVERS");
private static final String[] DGRAPH_ALPHA_HOSTS = CONFIG.getString("DGRAPH_HOSTS").split(",");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ public final class BackEnd extends AbstractBehavior<BackEnd.Event> {
private final String pgPassword;
private final String pgNotificationsDb;
private final String pgAuditDb;
private final String pgConfigurationDb;
private final PsqlNotifications psqlNotifications;
private LibMPI libMPI = null;
private String[] dgraphHosts = null;
Expand All @@ -40,6 +41,7 @@ private BackEnd(
final String sqlPassword,
final String sqlNotificationsDb,
final String sqlAuditDb,
final String sqlConfigurationDb,
final String kafkaBootstrapServers,
final String kafkaClientId) {
super(context);
Expand All @@ -53,6 +55,7 @@ private BackEnd(
this.pgPassword = sqlPassword;
this.pgNotificationsDb = sqlNotificationsDb;
this.pgAuditDb = sqlAuditDb;
this.pgConfigurationDb = sqlConfigurationDb;
psqlNotifications = new PsqlNotifications(sqlIP, sqlPort, sqlNotificationsDb, sqlUser, sqlPassword);
openMPI(kafkaBootstrapServers, kafkaClientId, debugLevel);
} catch (Exception e) {
Expand All @@ -72,6 +75,7 @@ public static Behavior<Event> create(
final String sqlPassword,
final String sqlNotificationsDb,
final String sqlAuditDb,
final String sqlConfigurationDb,
final String kafkaBootstrapServers,
final String kafkaClientId) {
return Behaviors.setup(context -> new BackEnd(level,
Expand All @@ -84,6 +88,7 @@ public static Behavior<Event> create(
sqlPassword,
sqlNotificationsDb,
sqlAuditDb,
sqlConfigurationDb,
kafkaBootstrapServers,
kafkaClientId));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public Behavior<Void> create() {
AppConfig.POSTGRESQL_PASSWORD,
AppConfig.POSTGRESQL_NOTIFICATIONS_DB,
AppConfig.POSTGRESQL_AUDIT_DB,
AppConfig.POSTGRESQL_CONFIGURATION_DB,
AppConfig.KAFKA_BOOTSTRAP_SERVERS,
"CLIENT_ID_API-" + UUID.randomUUID()), "BackEnd");
context.watch(backEnd);
Expand Down
1 change: 1 addition & 0 deletions JeMPI_Apps/JeMPI_Bootstrapper/boostrap.conf.sample
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ POSTGRESQL_USER=postgres
POSTGRESQL_PASSWORD=
POSTGRESQL_USERS_DB=
POSTGRESQL_NOTIFICATIONS_DB=
POSTGRESQL_CONFIGURATION_DB=
POSTGRESQL_AUDIT_DB=
POSTGRESQL_KC_TEST_DB=
KAFKA_BOOTSTRAP_SERVERS=127.0.0.1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public class BootstrapperConfig {
public final String POSTGRESQL_DATABASE;
public final String POSTGRESQL_USERS_DB;
public final String POSTGRESQL_NOTIFICATIONS_DB;
public final String POSTGRESQL_CONFIGURATION_DB;
public final String POSTGRESQL_AUDIT_DB;
public final String POSTGRESQL_KC_TEST_DB;
public final String KAFKA_BOOTSTRAP_SERVERS;
Expand All @@ -33,6 +34,7 @@ public BootstrapperConfig(final Config parsedConfig) {
POSTGRESQL_USERS_DB = parsedConfig.getString("POSTGRESQL_USERS_DB");
POSTGRESQL_NOTIFICATIONS_DB = parsedConfig.getString("POSTGRESQL_NOTIFICATIONS_DB");
POSTGRESQL_AUDIT_DB = parsedConfig.getString("POSTGRESQL_AUDIT_DB");
POSTGRESQL_CONFIGURATION_DB = parsedConfig.getString("POSTGRESQL_CONFIGURATION_DB");
POSTGRESQL_KC_TEST_DB = parsedConfig.getString("POSTGRESQL_KC_TEST_DB");

KAFKA_BOOTSTRAP_SERVERS = parsedConfig.getString("KAFKA_BOOTSTRAP_SERVERS");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,18 @@

import org.jembi.jempi.bootstrapper.data.DataBootstrapper;
import org.jembi.jempi.bootstrapper.data.utils.DataBootstraperConsts;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.sql.SQLException;
import java.sql.*;
import java.util.List;
import java.util.stream.Collectors;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Optional;

public class PostgresDataBootstrapper extends DataBootstrapper {

Expand Down Expand Up @@ -37,6 +41,7 @@ protected List<DBSchemaDetails> getAllDbSchemas() {
new DBSchemaDetails(this.loadedConfig.POSTGRESQL_USERS_DB, DataBootstraperConsts.POSTGRES_INIT_SCHEMA_USERS_DB),
new DBSchemaDetails(this.loadedConfig.POSTGRESQL_NOTIFICATIONS_DB, DataBootstraperConsts.POSTGRES_INIT_SCHEMA_NOTIFICATION_DB),
new DBSchemaDetails(this.loadedConfig.POSTGRESQL_AUDIT_DB, DataBootstraperConsts.POSTGRES_INIT_SCHEMA_AUDIT_DB),
new DBSchemaDetails(this.loadedConfig.POSTGRESQL_CONFIGURATION_DB, DataBootstraperConsts.POSTGRES_INIT_SCHEMA_CONFIGURATION_DB),
new DBSchemaDetails(this.loadedConfig.POSTGRESQL_KC_TEST_DB, null)
);
}
Expand Down Expand Up @@ -112,8 +117,57 @@ public Boolean deleteData() throws SQLException {
}

@Override
public Boolean resetAll() throws SQLException {
public Boolean resetAll() throws SQLException, IOException {
LOGGER.info("Resetting Postgres data and schemas.");
return this.deleteData() && this.deleteTables() && this.createSchema();
return this.deleteData() && this.deleteTables() && this.createSchema() && this.insertConfigurationData();
}

public Boolean insertConfigurationData() throws SQLException, IOException {
String configurationTable = "configuration";
String[][] configFiles = getConfigFiles(configurationTable);
String insertQuery = String.format("INSERT INTO %s (key, json) VALUES (?, ?::jsonb)", configurationTable);
postgresDALLib.runQuery(connection -> {
PreparedStatement pstmt = connection.prepareStatement(insertQuery);
for (String[] configFile : configFiles) {
String filePath = configFile[0];
String key = configFile[1];
try {
String jsonContent = readJsonFile(filePath);
pstmt.setString(1, key);
pstmt.setString(2, jsonContent);
pstmt.addBatch();
LOGGER.info("Added " + key + " configuration to batch");
} catch (IOException e) {
// Handle the exception, e.g., log it or throw a runtime exception
throw new RuntimeException("Error reading JSON file", e);
}
}
pstmt.executeBatch();
LOGGER.info("All configuration data inserted successfully");
return null;
}, true, this.loadedConfig.POSTGRESQL_CONFIGURATION_DB);
return true;
}

private String[][] getConfigFiles(final String configurationTable) {
String configDir = Optional.of(Paths.get("/app/conf_system"))
.filter(Files::exists)
.map(Path::toString)
.orElseGet(() -> System.getenv("SYSTEM_CSV_DIR"));

LOGGER.info("Inserting configuration data into " + configurationTable);
LOGGER.info("configDir " + configDir);
// Define an array of config file paths and their corresponding keys
String[][] configFiles = {
{configDir + "/config.json", "config"},
{configDir + "/config-api.json", "config-api"},
// Add more configuration files as needed
};
return configFiles;
}

private String readJsonFile(final String filePath) throws IOException {
Path path = Paths.get(filePath);
return Files.readString(path);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@ protected DataBootstraperConsts() { }
public static final String POSTGRES_INIT_SCHEMA_AUDIT_DB = "/data/postgres/audit-schema.sql";
public static final String POSTGRES_INIT_SCHEMA_NOTIFICATION_DB = "/data/postgres/notifications-schema.sql";
public static final String POSTGRES_INIT_SCHEMA_USERS_DB = "/data/postgres/users-schema.sql";
public static final String POSTGRES_INIT_SCHEMA_CONFIGURATION_DB = "/data/postgres/configuration-schema.sql";
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
CREATE TABLE IF NOT EXISTS configuration (
id SERIAL PRIMARY KEY,
key VARCHAR(255) NOT NULL,
json JSON NOT NULL,
dateCreated TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
dateUpdated TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
);

-- Create an index on the key column for faster lookups
CREATE INDEX IF NOT EXISTS idx_configuration_key ON configuration(key);

-- INSERT INTO configuration (key, json) VALUES
-- ('config', pg_read_file('/app/conf_system/config.json')::json);
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ public final class AppConfig {
public static final String POSTGRESQL_PASSWORD = CONFIG.getString("POSTGRESQL_PASSWORD");
public static final String POSTGRESQL_NOTIFICATIONS_DB = CONFIG.getString("POSTGRESQL_NOTIFICATIONS_DB");
public static final String POSTGRESQL_AUDIT_DB = CONFIG.getString("POSTGRESQL_AUDIT_DB");
public static final String POSTGRESQL_CONFIGURATION_DB = CONFIG.getString("POSTGRESQL_CONFIGURATION_DB");
public static final Integer CONTROLLER_HTTP_PORT = CONFIG.getInt("CONTROLLER_HTTP_PORT");
public static final String LINKER_IP = CONFIG.getString("LINKER_IP");
public static final Integer LINKER_HTTP_PORT = CONFIG.getInt("LINKER_HTTP_PORT");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ private void run() {
LOGGER.info("CONFIG: {} {} {} {} {}",
AppConfig.POSTGRESQL_NOTIFICATIONS_DB,
AppConfig.POSTGRESQL_AUDIT_DB,
AppConfig.POSTGRESQL_CONFIGURATION_DB,
AppConfig.KAFKA_BOOTSTRAP_SERVERS,
AppConfig.KAFKA_APPLICATION_ID,
AppConfig.KAFKA_CLIENT_ID);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public final class BackEnd extends AbstractBehavior<BackEnd.Event> {
private final String pgAuditDb;
private final PsqlNotifications psqlNotifications;
private final PsqlAuditTrail psqlAuditTrail;
private final String pgConfigurationDb;
private final String systemConfigDirectory;
private final String configReferenceFileName;
private final String configMasterFileName;
Expand All @@ -60,6 +61,7 @@ private BackEnd(
final String sqlPassword,
final String sqlNotificationsDb,
final String sqlAuditDb,
final String sqlConfigurationDb,
final String kafkaBootstrapServers,
final String kafkaClientId,
final String systemConfigDirectory,
Expand All @@ -77,6 +79,7 @@ private BackEnd(
this.pgPassword = sqlPassword;
this.pgNotificationsDb = sqlNotificationsDb;
this.pgAuditDb = sqlAuditDb;
this.pgConfigurationDb = sqlConfigurationDb;
this.systemConfigDirectory = systemConfigDirectory;
this.configReferenceFileName = configReferenceFileName;
this.configMasterFileName = configMasterFileName;
Expand All @@ -101,6 +104,7 @@ public static Behavior<Event> create(
final String sqlPassword,
final String sqlNotificationsDb,
final String sqlAuditDb,
final String sqlConfigurationDb,
final String kafkaBootstrapServers,
final String kafkaClientId,
final String systemConfigDirectory,
Expand All @@ -117,6 +121,7 @@ public static Behavior<Event> create(
sqlPassword,
sqlNotificationsDb,
sqlAuditDb,
sqlConfigurationDb,
kafkaBootstrapServers,
kafkaClientId,
systemConfigDirectory,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ public final class GlobalConstants {
public static final String TOPIC_AUDIT_TRAIL = "JeMPI-audit-trail";
public static final String TOPIC_NOTIFICATIONS = "JeMPI-notifications";
public static final String PSQL_TABLE_AUDIT_TRAIL = "audit_trail";

public static final String PSQL_TABLE_CONFIGURATION = "configuration";
/*
*
* HTTP SEGMENTS
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public final class AppConfig {
// public static final String POSTGRESQL_USER = CONFIG.getString("POSTGRESQL_USER");
// public static final String POSTGRESQL_PASSWORD = CONFIG.getString("POSTGRESQL_PASSWORD");
// public static final String POSTGRESQL_NOTIFICATIONS_DB = CONFIG.getString("POSTGRESQL_NOTIFICATIONS_DB");
public static final String POSTGRESQL_CONFIGURATION_DB = CONFIG.getString("POSTGRESQL_CONFIGURATION_DB");
public static final String KAFKA_BOOTSTRAP_SERVERS = CONFIG.getString("KAFKA_BOOTSTRAP_SERVERS");
public static final String KAFKA_APPLICATION_ID_INTERACTIONS = CONFIG.getString("KAFKA_APPLICATION_ID_INTERACTIONS");
public static final String KAFKA_APPLICATION_ID_MU = CONFIG.getString("KAFKA_APPLICATION_ID_MU");
Expand Down
2 changes: 1 addition & 1 deletion devops/linux/docker/backup_restore/postgres-backup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ DB_HOST="${POSTGRES_HOST:-localhost}"
BACKUP_PATH="${POSTGRES_BACKUP_DIRECTORY}/$BACKUP_DATE_TIME"
OLD_LOGS_DIR="${BACKUP_PATH}/old_logs" # Directory to store old logs

databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB")
databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB", "$POSTGRESQL_CONFIGURATION_DB")

# Check and Create Backup Directory and Old Logs Directory
[ ! -d "$BACKUP_PATH" ] && mkdir -p "$BACKUP_PATH"
Expand Down
2 changes: 1 addition & 1 deletion devops/linux/docker/backup_restore/postgres-restore.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ PGDATABASE="${POSTGRESQL_DATABASE}"

BACKUP_DIR="${POSTGRES_BACKUP_DIRECTORY}/$BACKUP_FOLDER_NAME"

databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB")
databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB", "$POSTGRESQL_CONFIGURATION_DB")


for db in "${databases[@]}"; do
Expand Down
1 change: 1 addition & 0 deletions devops/linux/docker/conf/env/conf-env-high-1-pc.template
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ export POSTGRESQL_USERS_DB=${POSTGRESQL_USERS_DB}
export POSTGRESQL_NOTIFICATIONS_DB=${POSTGRESQL_NOTIFICATIONS_DB}
export POSTGRESQL_AUDIT_DB=${POSTGRESQL_AUDIT_DB}
export POSTGRESQL_KC_TEST_DB=${POSTGRESQL_KC_TEST_DB}
export POSTGRESQL_CONFIGURATION_DB=${POSTGRESQL_CONFIGURATION_DB}

export KAFKA_SERVERS=${KAFKA_SERVERS}
export DGRAPH_HOSTS=${DGRAPH_HOSTS}
Expand Down
1 change: 1 addition & 0 deletions devops/linux/docker/conf/env/conf-env-low-1-pc.template
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ export POSTGRESQL_USERS_DB=${POSTGRESQL_USERS_DB}
export POSTGRESQL_NOTIFICATIONS_DB=${POSTGRESQL_NOTIFICATIONS_DB}
export POSTGRESQL_AUDIT_DB=${POSTGRESQL_AUDIT_DB}
export POSTGRESQL_KC_TEST_DB=${POSTGRESQL_KC_TEST_DB}
export POSTGRESQL_CONFIGURATION_DB=${POSTGRESQL_CONFIGURATION_DB}

export KAFKA_SERVERS=${KAFKA_SERVERS}
export DGRAPH_HOSTS=${DGRAPH_HOSTS}
Expand Down
1 change: 1 addition & 0 deletions devops/linux/docker/conf/env/create-env-linux-high-1.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ export POSTGRESQL_USERS_DB="users_db"
export POSTGRESQL_NOTIFICATIONS_DB="notifications_db"
export POSTGRESQL_AUDIT_DB="audit_db"
export POSTGRESQL_KC_TEST_DB="kc_test_db"
export POSTGRESQL_CONFIGURATION_DB="configuration_db"

export KAFKA_SERVERS="kafka-01:9092,kafka-02:9092,kafka-03:9092"
export DGRAPH_HOSTS="alpha-01,alpha-02,alpha-03"
Expand Down
1 change: 1 addition & 0 deletions devops/linux/docker/conf/env/create-env-linux-low-1.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ export POSTGRESQL_USERS_DB="users_db"
export POSTGRESQL_NOTIFICATIONS_DB="notifications_db"
export POSTGRESQL_AUDIT_DB="audit_db"
export POSTGRESQL_KC_TEST_DB="kc_test_db"
export POSTGRESQL_CONFIGURATION_DB="configuration_db"

export KAFKA_SERVERS="kafka-01:9092"
export DGRAPH_HOSTS="alpha-01"
Expand Down
1 change: 1 addition & 0 deletions devops/linux/docker/conf/env/create-env-wsl-low-1.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ export POSTGRESQL_USERS_DB="users_db"
export POSTGRESQL_NOTIFICATIONS_DB="notifications_db"
export POSTGRESQL_AUDIT_DB="audit_db"
export POSTGRESQL_KC_TEST_DB="kc_test_db"
export POSTGRESQL_CONFIGURATION_DB="configuration_db"

export KAFKA_SERVERS="kafka-01:9092"
export DGRAPH_HOSTS="alpha-01"
Expand Down
Loading