Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrated CheckRequiredToolsAreInstalled() function into checkDependenciesForExport() function #2071

Merged
merged 2 commits into from
Dec 19, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions yb-voyager/cmd/export.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ var useDebezium bool
var runId string
var excludeTableListFilePath string
var tableListFilePath string
var pgExportDependencies = []string{"pg_dump", "pg_restore", "psql"}
var pgExportCommands = []string{"pg_dump", "pg_restore", "psql"}

var exportCmd = &cobra.Command{
Use: "export",
Expand Down Expand Up @@ -392,18 +392,36 @@ func saveExportTypeInMSR() {
}

func checkDependenciesForExport() (binaryCheckIssues []string, err error) {
if source.DBType == POSTGRESQL {
var missingTools []string
switch source.DBType {
case POSTGRESQL:
sourceDBVersion := source.DB().GetVersion()
for _, binary := range pgExportDependencies {
for _, binary := range pgExportCommands {
_, binaryCheckIssue, err := srcdb.GetAbsPathOfPGCommandAboveVersion(binary, sourceDBVersion)
if err != nil {
return nil, err
} else if binaryCheckIssue != "" {
binaryCheckIssues = append(binaryCheckIssues, binaryCheckIssue)
}
}

missingTools = utils.CheckTools("strings")

case MYSQL:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor improvement: for mysql, oracle, you can skip the ora2pg check if it's live migration.
(add a todo for now, because as I see this function is called in lots of places - exportschema, exportdata, assess-migration)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a TODO statement and created a ticket:
#2102

missingTools = utils.CheckTools("ora2pg")

case ORACLE:
missingTools = utils.CheckTools("ora2pg", "sqlplus")

case YUGABYTEDB:
missingTools = utils.CheckTools("strings")

default:
return nil, fmt.Errorf("unknown source database type %q", source.DBType)
}

binaryCheckIssues = append(binaryCheckIssues, missingTools...)

if changeStreamingIsEnabled(exportType) || useDebezium {
// Check for java
javaIssue, err := checkJavaVersion()
Expand Down Expand Up @@ -489,7 +507,7 @@ func checkJavaVersion() (binaryCheckIssue string, err error) {
}

if majorVersion < MIN_REQUIRED_JAVA_VERSION {
return fmt.Sprintf("Java version %s is not supported. Please install Java version %d or higher", version, MIN_REQUIRED_JAVA_VERSION), nil
return fmt.Sprintf("java: required version >= %d; current version: %s", MIN_REQUIRED_JAVA_VERSION, version), nil
}

return "", nil
Expand Down
1 change: 0 additions & 1 deletion yb-voyager/cmd/exportData.go
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,6 @@ func exportData() bool {

clearMigrationStateIfRequired()
checkSourceDBCharset()
source.DB().CheckRequiredToolsAreInstalled()
saveSourceDBConfInMSR()
saveExportTypeInMSR()
err = InitNameRegistry(exportDir, exporterRole, &source, source.DB(), nil, nil, false)
Expand Down
1 change: 0 additions & 1 deletion yb-voyager/cmd/exportSchema.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,6 @@ func exportSchema() error {
}

checkSourceDBCharset()
source.DB().CheckRequiredToolsAreInstalled()
sourceDBVersion := source.DB().GetVersion()
source.DBVersion = sourceDBVersion
source.DBSize, err = source.DB().GetDatabaseSize()
Expand Down
4 changes: 0 additions & 4 deletions yb-voyager/src/srcdb/mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,6 @@ func (ms *MySQL) CheckSchemaExists() bool {
return true
}

func (ms *MySQL) CheckRequiredToolsAreInstalled() {
checkTools("ora2pg")
}

func (ms *MySQL) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) {
var rowCount int64
query := fmt.Sprintf("select count(*) from %s", tableName.AsQualifiedCatalogName())
Expand Down
4 changes: 0 additions & 4 deletions yb-voyager/src/srcdb/oracle.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,6 @@ func (ora *Oracle) CheckSchemaExists() bool {
return true
}

func (ora *Oracle) CheckRequiredToolsAreInstalled() {
checkTools("ora2pg", "sqlplus")
}

func (ora *Oracle) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) {
var rowCount int64
query := fmt.Sprintf("select count(*) from %s", tableName.ForUserQuery())
Expand Down
4 changes: 0 additions & 4 deletions yb-voyager/src/srcdb/postgres.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,10 +140,6 @@ func (pg *PostgreSQL) getTrimmedSchemaList() []string {
return trimmedList
}

func (pg *PostgreSQL) CheckRequiredToolsAreInstalled() {
checkTools("strings")
}

func (pg *PostgreSQL) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) {
var rowCount int64
query := fmt.Sprintf("select count(*) from %s", tableName.ForUserQuery())
Expand Down
1 change: 0 additions & 1 deletion yb-voyager/src/srcdb/srcdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ type SourceDB interface {
GetConnectionUriWithoutPassword() string
GetTableRowCount(tableName sqlname.NameTuple) (int64, error)
GetTableApproxRowCount(tableName sqlname.NameTuple) int64
CheckRequiredToolsAreInstalled()
GetVersion() string
GetAllTableNames() []*sqlname.SourceName
GetAllTableNamesRaw(schemaName string) ([]string, error)
Expand Down
15 changes: 0 additions & 15 deletions yb-voyager/src/srcdb/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,25 +17,10 @@ limitations under the License.
import (
"fmt"
"os"
"os/exec"
"path"
"strings"

log "github.com/sirupsen/logrus"

"github.com/yugabyte/yb-voyager/yb-voyager/src/utils"
)

func checkTools(tools ...string) {
for _, tool := range tools {
execPath, err := exec.LookPath(tool)
if err != nil {
utils.ErrExit("%q not found. Check if it is installed and included in the path.", tool)
}
log.Infof("Found %q", execPath)
}
}

func findAllExecutablesInPath(executableName string) ([]string, error) {
pathString := os.Getenv("PATH")
if pathString == "" {
Expand Down
4 changes: 0 additions & 4 deletions yb-voyager/src/srcdb/yugabytedb.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,6 @@ func (yb *YugabyteDB) Disconnect() {
}
}

func (yb *YugabyteDB) CheckRequiredToolsAreInstalled() {
checkTools("strings")
}

func (yb *YugabyteDB) GetTableRowCount(tableName sqlname.NameTuple) (int64, error) {
var rowCount int64
query := fmt.Sprintf("select count(*) from %s", tableName.ForUserQuery())
Expand Down
16 changes: 16 additions & 0 deletions yb-voyager/src/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"net"
"net/url"
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
Expand Down Expand Up @@ -718,3 +719,18 @@ func GetFinalReleaseVersionFromRCVersion(msrVoyagerFinalVersion string) (string,
}
return msrVoyagerFinalVersion, nil
}

// Return list of missing tools from the provided list of tools
func CheckTools(tools ...string) []string {
var missingTools []string
for _, tool := range tools {
execPath, err := exec.LookPath(tool)
if err != nil {
missingTools = append(missingTools, fmt.Sprintf("%s", tool))
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

missingTools = append(missingTools, tool) ??

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed it

} else {
log.Infof("Found %s at %s", tool, execPath)
}
}

return missingTools
}