Skip to content

bump aws sdk v2 to v3 #2254

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: development/2.13
Choose a base branch
from
Open

bump aws sdk v2 to v3 #2254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
0ab5d6f
bump zenko_tests from aws sdk v2 to v3
SylvainSenechal Aug 12, 2025
0e5d500
bump ctst tests aws sdk to higher versions
SylvainSenechal Oct 9, 2025
ccaa825
migrate backbeat tests to aws sdk v3
SylvainSenechal Oct 9, 2025
acadea3
migrate cloudserver tests to aws sdk v3
SylvainSenechal Oct 9, 2025
085d550
migrate node_tests client to aws sdk v3
SylvainSenechal Oct 9, 2025
b5553f5
migrate smoke_tests to aws sdk v3
SylvainSenechal Oct 9, 2025
54ba63d
migrate iam_policies tests to aws sdk v3
SylvainSenechal Oct 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions tests/ctst/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@
"scubaclient": "git+https://github.com/scality/scubaclient#^1.1.2"
},
"devDependencies": {
"@aws-sdk/client-iam": "^3.582.0",
"@aws-sdk/client-s3": "^3.583.0",
"@aws-sdk/client-sts": "^3.583.0",
"@aws-sdk/client-iam": "^3.901.0",
"@aws-sdk/client-s3": "^3.901.0",
"@aws-sdk/client-sts": "^3.901.0",
"@eslint/compat": "^1.1.1",
"cli-testing": "github:scality/cli-testing.git#1.2.4",
"eslint": "^9.9.1",
Expand Down
1,132 changes: 1,097 additions & 35 deletions tests/ctst/yarn.lock
View file
Open in desktop

Large diffs are not rendered by default.

247 changes: 102 additions & 145 deletions tests/zenko_tests/node_tests/VaultClient.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,130 +1,83 @@
const async = require('async');
const fs = require('fs');
const { IAM } = require('aws-sdk');
const {
IAMClient,
DetachUserPolicyCommand,
DetachRolePolicyCommand,
DeleteUserCommand,
DeleteRoleCommand,
DeletePolicyCommand,
paginateListAttachedUserPolicies,
paginateListAttachedRolePolicies,
paginateListUsers,
paginateListRoles,
paginateListPolicies,
} = require('@aws-sdk/client-iam');
const { NodeHttpHandler } = require('@aws-sdk/node-http-handler');
const vaultclient = require('vaultclient');
const https = require('https');

function _deleteAttachedUserPolicies(iamClient, userName, cb) {
let truncated = true;

async.whilst(
() => truncated,
done => iamClient.listAttachedUserPolicies(
{ UserName: userName },
(err, res) => {
if (err) {
return done(err);
}

truncated = res.IsTruncated;
return async.forEach(
res.AttachedPolicies,
(policy, next) => iamClient.detachUserPolicy({
PolicyArn: policy.PolicyArn,
UserName: userName,
}, next),
done,
);
},
),
cb,
);
async function _deleteAttachedUserPolicies(iamClient, userName) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if I should add a try catch to these functions. They are used for cleanup in before/after all tests

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMO no. Why do you think you should ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe its ok because these functions are just used for cleanup in after().
But if there are some errors without try catch, Im not sure the cleanup will be done properly

const paginator = paginateListAttachedUserPolicies({ client: iamClient }, { UserName: userName });
// eslint-disable-next-line no-restricted-syntax
for await (const page of paginator) {
if (page.AttachedPolicies && page.AttachedPolicies.length > 0) {
await Promise.all(page.AttachedPolicies.map(policy => iamClient.send(new DetachUserPolicyCommand({
PolicyArn: policy.PolicyArn,
UserName: userName,
}))));
}
}
}

function _deleteAttachedRolePolicies(iamClient, roleName, cb) {
let truncated = true;

async.whilst(
() => truncated,
done => iamClient.listAttachedRolePolicies(
{ RoleName: roleName },
(err, res) => {
if (err) {
return done(err);
}

truncated = res.IsTruncated;
return async.forEach(
res.AttachedPolicies,
(policy, next) => iamClient.detachRolePolicy({
PolicyArn: policy.PolicyArn,
RoleName: roleName,
}, next),
done,
);
},
),
cb,
);
async function _deleteAttachedRolePolicies(iamClient, roleName) {
const paginator = paginateListAttachedRolePolicies({ client: iamClient }, { RoleName: roleName });
// eslint-disable-next-line no-restricted-syntax
for await (const page of paginator) {
if (page.AttachedPolicies && page.AttachedPolicies.length > 0) {
await Promise.all(page.AttachedPolicies.map(policy => iamClient.send(new DetachRolePolicyCommand({
PolicyArn: policy.PolicyArn,
RoleName: roleName,
}))));
}
}
}

function _deleteUsers(iamClient, cb) {
let truncated = true;

async.whilst(
() => truncated,
done => iamClient.listUsers((err, res) => {
if (err) {
return done(err);
}

truncated = res.IsTruncated;
return async.forEach(
res.Users,
(user, next) => async.series([
next => _deleteAttachedUserPolicies(iamClient, user.UserName, next),
next => iamClient.deleteUser({ UserName: user.UserName }, next),
], next),
done,
);
}),
cb,
);
async function _deleteUsers(iamClient) {
const paginator = paginateListUsers({ client: iamClient }, {});
// eslint-disable-next-line no-restricted-syntax
for await (const page of paginator) {
if (page.Users && page.Users.length > 0) {
await Promise.all(page.Users.map(async user => {
await _deleteAttachedUserPolicies(iamClient, user.UserName);
await iamClient.send(new DeleteUserCommand({ UserName: user.UserName }));
}));
}
}
}

function _deleteRoles(iamClient, cb) {
let truncated = true;

async.whilst(
() => truncated,
done => iamClient.listRoles((err, res) => {
if (err) {
return done(err);
}

truncated = res.IsTruncated;
return async.forEach(
res.Roles,
(role, next) => async.series([
next => _deleteAttachedRolePolicies(iamClient, role.RoleName, next),
next => iamClient.deleteRole({ RoleName: role.RoleName }, next),
], next),
done,
);
}),
cb,
);
async function _deleteRoles(iamClient) {
const paginator = paginateListRoles({ client: iamClient }, {});
// eslint-disable-next-line no-restricted-syntax
for await (const page of paginator) {
if (page.Roles && page.Roles.length > 0) {
await Promise.all(page.Roles.map(async role => {
await _deleteAttachedRolePolicies(iamClient, role.RoleName);
await iamClient.send(new DeleteRoleCommand({ RoleName: role.RoleName }));
}));
}
}
}

function _deletePolicies(iamClient, cb) {
let truncated = true;

async.whilst(
() => truncated,
done => iamClient.listPolicies((err, res) => {
if (err) {
return done(err);
}

truncated = res.IsTruncated;
return async.forEach(
res.Policies,
(policy, next) => iamClient.deletePolicy({ PolicyArn: policy.Arn }, next),
done,
);
}),
cb,
);
async function _deletePolicies(iamClient) {
const paginator = paginateListPolicies({ client: iamClient }, { Scope: 'Local' });
// eslint-disable-next-line no-restricted-syntax
for await (const page of paginator) {
if (page.Policies && page.Policies.length > 0) {
await Promise.all(page.Policies.map(policy => iamClient.send(
new DeletePolicyCommand({ PolicyArn: policy.Arn }),
)));
}
}
}

class VaultClient {
Expand All @@ -140,40 +93,40 @@ class VaultClient {
static getIamClient(accessKey, secretKey, sessionToken) {
const endpoint = process.env.VAULT_ENDPOINT
|| 'http://localhost:8600';
let ca;
let httpOptions;
const info = {
endpoint,
region: 'us-east-1',
maxAttempts: 1,
tls: false,
credentials: {
accessKeyId: accessKey,
secretAccessKey: secretKey,
},
};

if (endpoint.startsWith('https://')) {
ca = fs.readFileSync(
const ca = fs.readFileSync(
process.env.VAULT_SSL_CA || '/conf/ca.crt',
'ascii',
);
httpOptions = {
agent: new https.Agent({
info.requestHandler = new NodeHttpHandler({
httpsAgent: new https.Agent({
ca: [ca],
}),
};
});
info.tls = true;
}
const info = {
endpoint,
httpOptions,
sslEnabled: httpOptions !== undefined,
region: 'us-east-1',
apiVersion: '2010-05-08',
signatureVersion: 'v4',
accessKeyId: accessKey,
secretAccessKey: secretKey,
maxRetries: 0,
};

if (sessionToken) {
info.sessionToken = sessionToken;
info.credentials.sessionToken = sessionToken;
}
return new IAM(info);
return new IAMClient(info);
}

/**
* Get endpoint information
*
* @return {object} Vault endpoint information
* @returns {object} Vault endpoint information
*/
static getEndpointInformation() {
let host = '127.0.0.1';
Expand Down Expand Up @@ -215,7 +168,7 @@ class VaultClient {
/**
* Get an admin client
*
* @return {vaultclient.Client} Vault client for admin calls
* @returns {vaultclient.Client} Vault client for admin calls
*/
static getAdminClient() {
const adminCredentials = {
Expand All @@ -241,17 +194,21 @@ class VaultClient {
* @param {vaultclient.Client} adminClient - Vault client for admin calls
* @param {object} iamClient - IAM client
* @param {string} accountName - account name
* @param {function} cb - callback
*
* @return {undefined}
* @returns {Promise<void>} Promise that resolves when account deletion is complete
*/
static deleteVaultAccount(adminClient, iamClient, accountName, cb) {
async.waterfall([
next => _deleteUsers(iamClient, next),
next => _deleteRoles(iamClient, next),
next => _deletePolicies(iamClient, next),
next => adminClient.deleteAccount(accountName, next),
], cb);
static async deleteVaultAccount(adminClient, iamClient, accountName) {
await _deleteUsers(iamClient);
await _deleteRoles(iamClient);
await _deletePolicies(iamClient);
await new Promise((resolve, reject) => {
adminClient.deleteAccount(accountName, err => {
if (err) {
return reject(err);
}
return resolve();
});
});
}
}
module.exports = VaultClient;
Loading
Loading