Skip to content

Commit

Permalink
Add project readme, config, gradle build, etc.
Browse files Browse the repository at this point in the history
  • Loading branch information
@jun-he
jun-he committed Apr 19, 2024
1 parent 541f99b commit cc4761a
Show file tree
Hide file tree
Showing 13 changed files with 797 additions and 0 deletions.
57 changes: 57 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
.gradle
.m2
build/
!gradle/wrapper/gradle-wrapper.jar
!**/src/main/**/build/
!**/src/test/**/build/

### IntelliJ IDEA ###
.idea
.idea/modules.xml
.idea/jarRepositories.xml
.idea/compiler.xml
.idea/libraries/
*.iws
*.iml
*.ipr
out/
!**/src/main/**/out/
!**/src/test/**/out/

### Eclipse ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
bin/
!**/src/main/**/bin/
!**/src/test/**/bin/

### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/

### VS Code ###
.vscode/

# OS generated files #
.DS_Store
ehthumbs.db
Icon?
Thumbs.db

# Editor Files #
################
*~
*.swp

# From local runners #
######################
tomcat.8080/
cockroach-data
38 changes: 38 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
Maestro
===================================
[Maestro](https://netflixtechblog.com/orchestrating-data-ml-workflows-at-scale-with-netflix-maestro-aaa2b41b800c#7d0f)
is a general-purpose workflow orchestrator that
provides a fully managed workflow-as-a-service (WAAS) to the data platform at Netflix.

It serves thousands of users, including data scientists, data engineers, machine learning engineers,
software engineers, content producers, and business analysts, for various use cases.
It schedules hundreds of thousands of workflows, millions of jobs every day
and operate with a strict SLO of less than 1 minute of scheduler introduced delay
even when there are spikes in the traffic.
Maestro is highly scalable and extensible to support existing and new use cases and offers enhanced usability to end users.

# Get started
## Prerequisite
- Git
- Java 8
- Gradle
- Docker


## Build it
- `./gradlew build`

# License
Copyright 2024 Netflix, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
88 changes: 88 additions & 0 deletions build.gradle
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
buildscript {
repositories {
mavenCentral()
maven {
url "https://plugins.gradle.org/m2/"
}
}
dependencies {
classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.10.0'
classpath "com.github.spotbugs.snom:spotbugs-gradle-plugin:6.+"
}
}

apply from: "${rootDir}/dependencies.gradle"

allprojects {
apply plugin: 'java'
apply plugin: 'jacoco'
apply plugin: 'com.diffplug.spotless'

repositories {
mavenCentral()
maven {
url "https://plugins.gradle.org/m2/"
}
}

spotless {
java {
googleJavaFormat()
removeUnusedImports() // removes any unused imports
}
}

java {
toolchain {
languageVersion = JavaLanguageVersion.of(8)
}
}
}

configure(allprojects - project(':cockroachdb-persistence') - project(':netflix-sel')) {
apply plugin: 'checkstyle'
apply plugin: 'pmd'
apply plugin: 'com.github.spotbugs'

apply plugin: 'java-library'
apply plugin: 'java-test-fixtures'

checkstyle {
checkstyleMain.enabled = true
checkstyleMain.source = fileTree("src/main/java")
checkstyleTest.enabled = false
checkstyleTestFixtures.enabled = true
}

pmd {
toolVersion = "6.21.0"
pmdTest.enabled = false
pmdTestFixtures.enabled = true
consoleOutput = true
rulesMinimumPriority = 5
ruleSetFiles = files("${rootDir}/config/pmd/ruleset.xml")
ruleSets = []
}

spotbugs {
ignoreFailures = true
}

group = 'com.netflix.maestro'

// Print out full stack traces when our tests fail to assist debugging (e.g., when scanning Jenkins console output)
tasks.withType(Test).configureEach {
useJUnitPlatform()
testLogging {
exceptionFormat = 'full'
}
}

dependencies {
compileOnly lombokDep
annotationProcessor lombokDep

testCompileOnly lombokDep
testAnnotationProcessor lombokDep
}
}
201 changes: 201 additions & 0 deletions config/checkstyle/checkstyle.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,201 @@
<?xml version="1.0"?>
<!DOCTYPE module PUBLIC
"-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
"https://checkstyle.org/dtds/configuration_1_3.dtd">

<module name="Checker">

<!-- Checks that a package-info.java file exists for each package. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html#JavadocPackage -->
<!--
<module name="JavadocPackage">
<property name="allowLegacy" value="true"/>
</module>
-->

<!-- Checks whether files end with a new line. -->
<!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile -->
<module name="NewlineAtEndOfFile"/>

<!-- Checks that property files contain the same keys. -->
<!-- See http://checkstyle.sf.net/config_misc.html#Translation -->
<module name="Translation"/>

<!-- Checks for Size Violations. -->
<!-- See http://checkstyle.sf.net/config_sizes.html -->
<module name="FileLength"/>

<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
<module name="FileTabCharacter"/>

<!-- Miscellaneous other checks. -->
<!-- See http://checkstyle.sf.net/config_misc.html -->
<module name="RegexpSingleline">
<property name="format" value="\s+$"/>
<property name="minimum" value="0"/>
<property name="maximum" value="0"/>
<property name="message" value="Line has trailing spaces."/>
<property name="severity" value="info"/>
</module>

<module name="SuppressionFilter">
<property name="file" value="${config_loc}/suppressions.xml"/>
</module>

<module name="SuppressWarningsFilter"/>

<module name="LineLength">
<!-- what is a good max value? -->
<property name="max" value="125"/>
<!-- ignore lines like "$File: //depot/... $" -->
<property name="ignorePattern" value="\$File.*\$"/>
<property name="severity" value="info"/>
</module>

<module name="TreeWalker">
<module name="MultipleStringLiterals"/>
<module name="StringLiteralEquality"/>

<!-- Checks for Javadoc comments. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html -->
<module name="JavadocMethod">
<property name="accessModifiers" value="package"/>
<property name="allowMissingParamTags" value="true"/>
<property name="allowMissingReturnTag" value="true"/>
</module>
<module name="JavadocType">
<property name="scope" value="public"/>
</module>
<module name="JavadocVariable">
<property name="scope" value="public"/>
</module>
<module name="JavadocStyle">
<property name="checkEmptyJavadoc" value="true"/>
</module>

<!-- Checks for Naming Conventions. -->
<!-- See http://checkstyle.sf.net/config_naming.html -->
<module name="ConstantName"/>
<module name="LocalFinalVariableName"/>
<module name="LocalVariableName"/>
<module name="MemberName"/>
<module name="MethodName"/>
<module name="PackageName"/>
<module name="ParameterName"/>
<module name="StaticVariableName"/>
<module name="TypeName"/>

<!-- Checks for imports -->
<!-- See http://checkstyle.sf.net/config_import.html -->
<module name="AvoidStarImport"/>
<module name="IllegalImport"/> <!-- defaults to sun.* packages -->
<module name="RedundantImport"/>
<module name="UnusedImports"/>


<!-- Checks for Size Violations. -->
<!-- See http://checkstyle.sf.net/config_sizes.html -->

<module name="MethodLength"/>
<module name="ParameterNumber">
<property name="max" value="32"/>
</module>


<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
<module name="EmptyForIteratorPad">
<property name="option" value="space"/>
</module>
<module name="GenericWhitespace"/>
<module name="MethodParamPad"/>
<module name="NoWhitespaceAfter"/>
<module name="NoWhitespaceBefore"/>
<module name="OperatorWrap"/>
<module name="ParenPad"/>
<module name="TypecastParenPad"/>
<module name="WhitespaceAfter"/>
<module name="WhitespaceAround">
<!--https://github.com/checkstyle/checkstyle/issues/6106-->
<property name="allowEmptyLambdas" value="true"/>
<property name="allowEmptyMethods" value="true"/>
<property name="allowEmptyConstructors" value="true"/>
<property name="allowEmptyTypes" value="true"/>
</module>

<!-- Modifier Checks -->
<!-- See http://checkstyle.sf.net/config_modifiers.html -->
<module name="ModifierOrder"/>
<module name="RedundantModifier"/>


<!-- Checks for blocks. You know, those {}'s -->
<!-- See http://checkstyle.sf.net/config_blocks.html -->
<module name="AvoidNestedBlocks"/>
<module name="EmptyBlock">
<property name="option" value="text"/>
</module>
<module name="LeftCurly"/>
<module name="NeedBraces"/>
<module name="RightCurly"/>


<!-- Checks for common coding problems -->
<!-- See http://checkstyle.sf.net/config_coding.html -->
<!-- <module name="AvoidInlineConditionals"/> -->
<module name="EmptyStatement"/>
<module name="EqualsHashCode"/>
<module name="HiddenField">
<property name="ignoreConstructorParameter" value="true"/>
<property name="ignoreSetter" value="true"/>
<property name="severity" value="warning"/>
</module>
<module name="IllegalInstantiation"/>
<module name="InnerAssignment"/>
<module name="MagicNumber">
<property name="ignoreNumbers" value="-2, -1, 0, 1, 2"/>
</module>
<module name="MissingSwitchDefault"/>
<module name="SimplifyBooleanExpression"/>
<module name="SimplifyBooleanReturn"/>

<!-- Checks for class design -->
<!-- See http://checkstyle.sf.net/config_design.html -->
<!-- <module name="DesignForExtension"/> -->
<module name="FinalClass"/>
<module name="HideUtilityClassConstructor"/>
<module name="InterfaceIsType"/>
<module name="VisibilityModifier"/>


<!-- Miscellaneous other checks. -->
<!-- See http://checkstyle.sf.net/config_misc.html -->
<module name="ArrayTypeStyle"/>
<!-- <module name="FinalParameters"/> -->
<module name="TodoComment">
<property name="format" value="TODO"/>
<property name="severity" value="info"/>
</module>
<module name="UpperEll"/>

<!-- Enable suppression comments -->
<module name="SuppressionCommentFilter">
<property name="offCommentFormat" value="CHECKSTYLE IGNORE\s+(\S+)"/>
<property name="onCommentFormat" value="CHECKSTYLE END IGNORE\s+(\S+)"/>
<property name="checkFormat" value="$1"/>
</module>
<module name="SuppressWithNearbyCommentFilter">
<!-- Syntax is "SUPPRESS CHECKSTYLE name" -->
<property name="commentFormat" value="SUPPRESS CHECKSTYLE (\w+)"/>
<property name="checkFormat" value="$1"/>
<property name="influenceFormat" value="1"/>
</module>

<module name="SuppressWarningsHolder"/>


</module>
<!-- Detects duplicated keys in properties files. -->
<module name="UniqueProperties"/>
</module>
8 changes: 8 additions & 0 deletions config/checkstyle/suppressions.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<?xml version="1.0"?>
<!DOCTYPE suppressions PUBLIC
"-//Checkstyle//DTD SuppressionFilter Configuration 1.2//EN"
"https://checkstyle.org/dtds/suppressions_1_2.dtd">
<suppressions>
<suppress checks="." files="com.netflix.maestro.engine.*.package-info.java" />
<suppress checks="OperatorWrap" files="com.netflix.maestro.engine.tasks.MaestroEndTask.java" />
</suppressions>
Loading

0 comments on commit cc4761a

Please sign in to comment.