Merge pull request #1 from mdevilliers/v19_dev

Release for Mesos 0.19

.gitignore

@@ -7,5 +7,6 @@ ebin
 priv
 erl_crash.dump
 include/mesos_pb.hrl
+include/containerizer_pb.hrl
 src/mesos_pb.erl
-
+src/containerizer_pb.erl

proto/containerizer.proto

@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import "mesos.proto";
+
+package mesos.containerizer;
+
+option java_package = "org.apache.mesos.containerizer";
+option java_outer_classname = "Protos";
+
+
+/**
+ * Encodes the launch command sent to the external containerizer
+ * program.
+ */
+message Launch {
+  required ContainerID container_id = 1;
+  optional TaskInfo task_info = 2;
+  optional ExecutorInfo executor_info = 3;
+  optional string directory = 4;
+  optional string user = 5;
+  optional SlaveID slave_id = 6;
+  optional string slave_pid = 7;
+  optional bool checkpoint = 8;
+}
+
+
+/**
+ * Encodes the update command sent to the external containerizer
+ * program.
+ */
+message Update {
+  required ContainerID container_id = 1;
+  repeated Resource resources = 2;
+}
+
+
+/**
+ * Encodes the wait command sent to the external containerizer
+ * program.
+ */
+message Wait {
+  required ContainerID container_id = 1;
+}
+
+
+/**
+ * Encodes the destroy command sent to the external containerizer
+ * program.
+ */
+message Destroy {
+  required ContainerID container_id = 1;
+}
+
+
+/**
+ * Encodes the usage command sent to the external containerizer
+ * program.
+ */
+message Usage {
+  required ContainerID container_id = 1;
+}
+
+
+/**
+ * Information about a container termination, returned by the
+ * containerizer to the slave.
+ */
+message Termination {
+  // A container may be killed if it exceeds its resources; this will
+  // be indicated by killed=true and described by the message string.
+  required bool killed = 1;
+  required string message = 2;
+
+  // Exit status of the process.
+  optional int32 status = 3;
+}
+
+
+/**
+ * Information on all active containers returned by the containerizer
+ * to the slave.
+ */
+message Containers {
+  repeated ContainerID containers = 1;
+}

proto/mesos.proto

@@ -94,20 +94,27 @@ message ContainerID {
 
 
 /**
- * Describes a framework. If the user field is set to an empty string
- * Mesos will automagically set it to the current user. Note that the
- * ID is only available after a framework has registered, however, it
- * is included here in order to facilitate scheduler failover (i.e.,
- * if it is set then the MesosSchedulerDriver expects the scheduler is
- * performing failover). The amount of time that the master will wait
- * for the scheduler to failover before removing the framework is
- * specified by failover_timeout.
- * If checkpoint is set, framework pid, executor pids and status updates
- * are checkpointed to disk by the slaves.
- * Checkpointing allows a restarted slave to reconnect with old executors
- * and recover status updates, at the cost of disk I/O.
- * The role field is used to group frameworks for allocation decisions,
- * depending on the allocation policy being used.
+ * Describes a framework. The user field is used to determine the
+ * Unix user that an executor/task should be launched as. If the user
+ * field is set to an empty string Mesos will automagically set it
+ * to the current user. Note that the ID is only available after a
+ * framework has registered, however, it is included here in order to
+ * facilitate scheduler failover (i.e., if it is set then the
+ * MesosSchedulerDriver expects the scheduler is performing failover).
+ * The amount of time that the master will wait for the scheduler to
+ * failover before removing the framework is specified by
+ * failover_timeout. If checkpoint is set, framework pid, executor
+ * pids and status updates are checkpointed to disk by the slaves.
+ * Checkpointing allows a restarted slave to reconnect with old
+ * executors and recover status updates, at the cost of disk I/O.
+ * The role field is used to group frameworks for allocation
+ * decisions, depending on the allocation policy being used.
+ * If the hostname field is set to an empty string Mesos will
+ * automagically set it to the current hostname.
+ * The principal field should match the credential the framework uses
+ * in authentication. This field is used for framework API rate
+ * exporting and limiting and should be set even if authentication is
+ * not enabled if these features are desired.
  */
 message FrameworkInfo {
   required string user = 1;
@@ -116,6 +123,63 @@ message FrameworkInfo {
   optional double failover_timeout = 4 [default = 0.0];
   optional bool checkpoint = 5 [default = false];
   optional string role = 6 [default = "*"];
+  optional string hostname = 7;
+  optional string principal = 8;
+}
+
+
+/**
+ * Describes a health check for a task or executor (or any arbitrary
+ * process/command). A "strategy" is picked by specifying one of the
+ * optional fields, currently only 'http' is supported. Specifying
+ * more than one strategy is an error.
+ */
+message HealthCheck {
+  // Describes an HTTP health check.
+  message HTTP {
+    // Port to send the HTTP request.
+    required uint32 port = 1;
+
+    // HTTP request path.
+    optional string path = 2 [default = "/"];
+
+    // TODO(benh): Implement:
+    // Whether or not to use HTTPS.
+    // optional bool ssl = 3 [default = false];
+
+    // Expected response statuses. Not specifying any statuses implies
+    // that any returned status is acceptable.
+    repeated uint32 statuses = 4;
+
+    // TODO(benh): Include an 'optional bytes data' field for checking
+    // for specific data in the response.
+  }
+
+  optional HTTP http = 1;
+
+  // TODO(benh): Consider adding a URL health check strategy which
+  // allows doing something similar to the HTTP strategy but
+  // encapsulates all the details in a single string field.
+
+  // TODO(benh): Other possible health check strategies could include
+  // one for TCP/UDP or a "command". A "command" could be running a
+  // (shell) command to check the healthiness. We'd need to determine
+  // what arguments (or environment variables) we'd want to set so
+  // that the command could do it's job (i.e., do we want to expose
+  // the stdout/stderr and/or the pid to make checking for healthiness
+  // easier).
+
+  // Amount of time to wait until starting the health checks.
+  optional double delay_seconds = 2 [default = 15.0];
+
+  // Interval between health checks.
+  optional double interval_seconds = 3 [default = 10.0];
+
+  // Amount of time to wait for the health check to complete.
+  optional double timeout_seconds = 4 [default = 20.0];
+
+  // Number of consecutive failures until considered unhealthy.
+  optional uint32 failures = 5 [default = 3];
 }
 
 
@@ -125,18 +189,53 @@ message FrameworkInfo {
  * uri is set, executable file permission is set on the downloaded file.
  * Otherwise, if the downloaded file has a recognized archive extension
  * (currently [compressed] tar and zip) it is extracted into the executor's
- * working directory.  In addition, any environment variables are set before
- * executing the command (so they can be used to "parameterize" your command).
+ * working directory. This extraction can be disabled by setting `extract` to
+ * false. In addition, any environment variables are set before executing
+ * the command (so they can be used to "parameterize" your command).
  */
 message CommandInfo {
   message URI {
     required string value = 1;
     optional bool executable = 2;
+    optional bool extract = 3 [default = true];
   }
 
+  // Describes a container.
+  // Not all containerizers currently implement ContainerInfo, so it
+  // is possible that a launched task will fail due to supplying this
+  // attribute.
+  // NOTE: The containerizer API is currently in an early beta or
+  // even alpha state. Some details, like the exact semantics of an
+  // "image" or "options" are not yet hardened.
+  // TODO(tillt): Describe the exact scheme and semantics of "image"
+  // and "options".
+  message ContainerInfo {
+    // URI describing the container image name.
+    required string image = 1;
+
+    // Describes additional options passed to the containerizer.
+    repeated string options = 2;
+  }
+
+  // NOTE: MesosContainerizer does currently not support this
+  // attribute and tasks supplying a 'container' will fail.
+  optional ContainerInfo container = 4;
+
   repeated URI uris = 1;
+
   optional Environment environment = 2;
+
+  // Actual command (i.e., 'echo hello world').
   required string value = 3;
+
+  // Enables executor and tasks to run as a specific user. If the user
+  // field is present both in FrameworkInfo and here, the CommandInfo
+  // user value takes precedence.
+  optional string user = 5;
+
+  // A health check for the command (currently in *alpha* and initial
+  // support will only be for TaskInfo's that have a CommandInfo).
+  optional HealthCheck health_check = 6;
 }
 
 
@@ -284,7 +383,7 @@ message ResourceStatistics {
   optional double cpus_system_time_secs = 3;
 
   // Number of CPUs allocated.
-  required double cpus_limit = 4;
+  optional double cpus_limit = 4;
 
   // cpu.stat on process throttling (for contention issues).
   optional uint32 cpus_nr_periods = 7;
@@ -461,12 +560,98 @@ message Parameters {
 /**
  * Credential used for authentication.
  *
- * NOTE: The 'principal' is used for authenticating the framework with
- * the master. This is different from 'FrameworkInfo.user'
+ * NOTE: The 'principal' is used for authenticating the framework or slave
+ * with the master. This is different from 'FrameworkInfo.user'
  * which is used to determine the user under which the framework's
  * executors/tasks are run.
  */
 message Credential {
   required string principal = 1;
   optional bytes secret = 2;
 }
+
+
+/**
+ * ACLs used for authorization.
+ */
+message ACL {
+
+  // Entity is used to describe a subject(s) or an object(s) of an ACL.
+  // NOTE:
+  // To allow everyone access to an Entity set its type to 'ANY'.
+  // To deny access to an Entity set its type to 'NONE'.
+  message Entity {
+    enum Type {
+      SOME = 0;
+      ANY = 1;
+      NONE = 2;
+    }
+    optional Type type = 1 [default = SOME];
+    repeated string values = 2; // Ignored for ANY/NONE.
+  }
+
+  // ACLs.
+  message RunTasks {
+    // Subjects.
+    required Entity principals = 1; // Framework principals.
+
+    // Objects.
+    required Entity users = 2; // Users to run the tasks/executors as.
+  }
+
+  message ReceiveOffers {
+    // Subjects.
+    required Entity principals = 1; // Framework principals.
+
+    // Objects.
+    required Entity roles = 2; // Resource roles that can be offered.
+  }
+
+  message HTTPGet {
+    // Subjects (At least one of these should be set).
+    optional Entity usernames = 1; // HTTP authentication based usernames.
+    optional Entity ips = 2;
+    optional Entity hostnames = 3;
+
+    // Objects.
+    required Entity urls = 4;
+  }
+
+  message HTTPPut {
+    // Subjects (At least one of these should be set).
+    optional Entity usernames = 1; // HTTP authentication based usernames.
+    optional Entity ips = 2;
+    optional Entity hostnames = 3;
+
+    // Objects.
+    required Entity urls = 4;
+  }
+}
+
+
+/*
+ * Collection of ACL.
+ *
+ * Each authorization request is evaluated against the ACLs in the order
+ * they are defined.
+ *
+ * For simplicity, the ACLs for a given action are not aggregated even
+ * when they have the same subjects or objects. The first ACL that
+ * matches the request determines whether that request should be
+ * permitted or not. An ACL matches iff both the subjects
+ * (e.g., clients, principals) and the objects (e.g., urls, users,
+ * roles) of the ACL match the request.
+ *
+ * If none of the ACLs match the request, the 'permissive' field
+ * determines whether the request should be permitted or not.
+ *
+ * TODO(vinod): Do aggregation of ACLs when possible.
+ *
+ */
+message ACLs {
+  optional bool permissive = 1 [default = true];
+  repeated ACL.RunTasks run_tasks = 2;
+  repeated ACL.ReceiveOffers receive_offers = 3;
+  repeated ACL.HTTPGet http_get = 4;
+  repeated ACL.HTTPPut http_put = 5;
+}

readme.md

@@ -32,7 +32,11 @@ Getting started
 
 The vagrant file I use for dev testing is at https://github.com/mdevilliers/vagrant-mesos-development-environment. It can be pointed at the version of mesos you wish to develop against plus installs erlang, gcc, git ect.
 
-Important : Install protobuf version 2.5 - mesos 0.18.0 requires a release >= 2.5
+Install the protobuf tools.
+
+```
+sudo apt-get install libprotobuf-dev protobuf-compiler
+```
 
 ```
 git clone .....