Merge pull request juju#6957 from nskaggs/add-skip-flaky-develop

Add skip flaky develop

## Description of change
This change adds a new flag for use with intermittent test failures. Once CI has identified an intermittent failure, which is not otherwise able to be fixed at the present time, the flag can be added to the testcase in order to skip it.

This change is needed to ensure the CI test results are actionable and consistent. You are encourage to use the arch specific skips as needed (as I have done on a few below, for example, skipping only on windows).

## QA steps

Build and run the unit tests. Ensure the skipped tests indeed show as skips.

## Documentation changes

None.

## Bug reference

The skipped tests all contain links to bug numbers, which link to CI issues. They can be used to fix the tests and remove the skips.

apiserver/pinger_test.go

@@ -94,6 +94,7 @@ func (s *pingerSuite) TestClientNoNeedToPing(c *gc.C) {
 }
 
 func (s *pingerSuite) TestAgentConnectionShutsDownWithNoPing(c *gc.C) {
+	coretesting.SkipFlaky(c, "lp:1627086")
 	server, clock := s.newServerWithTestClock(c)
 	conn, _ := s.OpenAPIAsNewMachine(c, server)
 
@@ -102,6 +103,7 @@ func (s *pingerSuite) TestAgentConnectionShutsDownWithNoPing(c *gc.C) {
 }
 
 func (s *pingerSuite) TestAgentConnectionDelaysShutdownWithPing(c *gc.C) {
+	coretesting.SkipFlaky(c, "lp:1632485")
 	server, clock := s.newServerWithTestClock(c)
 	conn, _ := s.OpenAPIAsNewMachine(c, server)
 

cmd/juju/application/bundle_test.go

@@ -443,6 +443,7 @@ func (s *BundleDeployCharmStoreSuite) TestDeployBundleInvalidSpace(c *gc.C) {
 }
 
 func (s *BundleDeployCharmStoreSuite) TestDeployBundleWatcherTimeout(c *gc.C) {
+	coretesting.SkipFlaky(c, "lp:1625213")
 	// Inject an "AllWatcher" that never delivers a result.
 	ch := make(chan struct{})
 	defer close(ch)

cmd/jujud/agent/machine_charms_test.go

@@ -45,6 +45,7 @@ func (s *MachineWithCharmsSuite) TearDownTest(c *gc.C) {
 }
 
 func (s *MachineWithCharmsSuite) TestManageModelRunsCharmRevisionUpdater(c *gc.C) {
+	coretesting.SkipFlaky(c, "lp:1466514")
 	m, _, _ := s.primeAgent(c, state.JobManageModel)
 
 	s.SetupScenario(c)

cmd/jujud/agent/machine_test.go

@@ -1037,6 +1037,7 @@ func (s *MachineSuite) TestMachineAgentDoesNotRunsCertificateUpdateWorkerForNonC
 }
 
 func (s *MachineSuite) TestCertificateUpdateWorkerUpdatesCertificate(c *gc.C) {
+	coretesting.SkipFlaky(c, "lp:1466514")
 	// Set up the machine agent.
 	m, _, _ := s.primeAgent(c, state.JobManageModel)
 	a := s.newAgent(c, m)

cmd/jujud/agent/unit_test.go

@@ -412,6 +412,7 @@ func (s *UnitSuite) TestChangeConfig(c *gc.C) {
 }
 
 func (s *UnitSuite) TestWorkers(c *gc.C) {
+	coretesting.SkipIfWindowsBug(c, "lp:1610993")
 	tracker := NewEngineTracker()
 	instrumented := TrackUnits(c, tracker, unitManifolds)
 	s.PatchValue(&unitManifolds, instrumented)

provider/common/bootstrap_test.go

@@ -387,6 +387,7 @@ func (ac *addressesChange) Addresses() ([]network.Address, error) {
 }
 
 func (s *BootstrapSuite) TestWaitSSHRefreshAddresses(c *gc.C) {
+	coretesting.SkipIfWindowsBug(c, "lp:1604961")
 	ctx := coretesting.Context(c)
 	_, err := common.WaitSSH(ctx.Stderr, nil, ssh.DefaultClient, "", &addressesChange{addrs: [][]string{
 		nil,

testing/base.go

@@ -101,6 +101,11 @@ func SkipIfWindowsBug(c *gc.C, bugID string) {
 	}
 }
 
+// SkipFlaky skips the test if there is an open bug for intermittent test failures
+func SkipFlaky(c *gc.C, bugID string) {
+	c.Skip(fmt.Sprintf("Test disabled until flakiness is fixed - see bug %s", bugID))
+}
+
 // SetInitialFeatureFlags sets the feature flags to be in effect for
 // the next call to SetUpTest.
 func (s *JujuOSEnvSuite) SetInitialFeatureFlags(flags ...string) {

worker/certupdater/certupdater_test.go

@@ -133,6 +133,7 @@ func (s *CertUpdaterSuite) TestStartStop(c *gc.C) {
 }
 
 func (s *CertUpdaterSuite) TestAddressChange(c *gc.C) {
+	coretesting.SkipFlaky(c, "lp:1466514")
 	var srvCert *x509.Certificate
 	updated := make(chan struct{})
 	setter := func(info params.StateServingInfo, dying <-chan struct{}) error {