diff --git a/configure.ac b/configure.ac index 47fee3d..3fdf3f0 100644 --- a/configure.ac +++ b/configure.ac @@ -44,6 +44,20 @@ AS_IF([test "x$with_lustre" != xno], [ AC_MSG_FAILURE([--with-lustre was given, not found]) fi ]) + AC_CHECK_HEADERS([lustre/liblustreapi.h], [AC_DEFINE([HAVE_LUSTREAPI],[1],[Have LustreAPI]) with_lustreapi=true], [ + if test "x$with_lustre" != xcheck; then + AC_MSG_ERROR([--with-lustre was given, not found]) + fi + ]) + +]) +AM_CONDITIONAL([USE_LUSTRE], [test x$with_lustre = xyes]) +AM_COND_IF([USE_LUSTRE],[ + AC_DEFINE([USE_LUSTRE], [], [Build LUSTRE support]) +]) +AM_CONDITIONAL([USE_LUSTREAPI], [test x$with_lustreapi = xtrue]) +AM_COND_IF([USE_LUSTREAPI],[ + AC_DEFINE([USE_LUSTREAPI], [], [Build LUSTRE support]) ]) # POSIX IO support diff --git a/doc/USER_GUIDE b/doc/USER_GUIDE index 5ef5027..9fe83dc 100644 --- a/doc/USER_GUIDE +++ b/doc/USER_GUIDE @@ -341,6 +341,14 @@ LUSTRE-SPECIFIC: * lustreIgnoreLocks - disable lustre range locking [0] + * lustreUseAllOsts - set specifically OSTs as starting OST for the test file(s) [default=-1] : + * -1 = let Lustre choose OST + * 0 = use all available OSTs of Lustre file system as starting OSTs + * n = use the first n OSTs available of Lustre file system as starting OSTs + * [l-m,n] = use the specified OSTs as starting OSTs + Note: if Lustre file system has 4 OSTs, the option "lustreStripeCount=1,lustreUseAllOsts=4" + will write one file one each OST + *********************** * 5. VERBOSITY LEVELS * diff --git a/src/Makefile.am b/src/Makefile.am index df034ff..cd739fd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,9 +7,14 @@ ior_SOURCES = ior.c utilities.c parse_options.c ior_SOURCES += ior.h utilities.h parse_options.h aiori.h iordef.h ior_LDADD = +if USE_LUSTREAPI +ior_LDADD += -llustreapi +endif + if USE_POSIX_AIORI ior_SOURCES += aiori-POSIX.c endif + if USE_MPIIO_AIORI ior_SOURCES += aiori-MPIIO.c endif diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 63791d9..c832505 100644 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -35,6 +35,7 @@ #include "ior.h" #include "aiori.h" #include "iordef.h" +#include "utilities.h" #ifndef open64 /* necessary for TRU64 -- */ #define open64 open /* unlikely, but may pose */ @@ -59,6 +60,7 @@ static void POSIX_SetVersion(IOR_param_t *); static void POSIX_Fsync(void *, IOR_param_t *); static IOR_offset_t POSIX_GetFileSize(IOR_param_t *, MPI_Comm, char *); + /************************** D E C L A R A T I O N S ***************************/ ior_aiori_t posix_aiori = { @@ -98,6 +100,11 @@ static void *POSIX_Create(char *testFileName, IOR_param_t * param) { int fd_oflag = O_BINARY; int *fd; +#ifdef HAVE_LUSTRE_LUSTRE_USER_H + char path[PATH_MAX] = ""; + int ost_count = 0; + int ost_list[MAX_OST_COUNT]; +#endif fd = (int *)malloc(sizeof(int)); if (fd == NULL) @@ -123,7 +130,30 @@ static void *POSIX_Create(char *testFileName, IOR_param_t * param) /* Setup Lustre IOCTL striping pattern structure */ opts.lmm_magic = LOV_USER_MAGIC; opts.lmm_stripe_size = param->lustre_stripe_size; - opts.lmm_stripe_offset = param->lustre_start_ost; + if(param->lustre_use_all_osts == -1){ + opts.lmm_stripe_offset = param->lustre_start_ost; + } +#ifdef HAVE_LUSTRE_LIBLUSTREAPI_H + /* Set lustre_start_ost if asked */ + else { + /* Get information about available FS's OSTs */ + getcwd(path, sizeof(path)); + get_active_ost_list(path,&ost_count,ost_list); + /* In the case where all OSTs will be used */ + if(param->lustre_use_all_osts == 0) { + opts.lmm_stripe_offset = ost_list[rank%ost_count]; + } else { + /* Only a number of OSTs is specified */ + if(param->lustre_ost_list[0] < 0) { + opts.lmm_stripe_offset = ost_list[rank%param->lustre_use_all_osts]; + } + /* a list of OSTs is specified */ + else { + opts.lmm_stripe_offset = param->lustre_ost_list[rank%param->lustre_use_all_osts]; + } + } + } +#endif opts.lmm_stripe_count = param->lustre_stripe_count; /* File needs to be opened O_EXCL because we cannot set @@ -347,3 +377,5 @@ static IOR_offset_t POSIX_GetFileSize(IOR_param_t * test, MPI_Comm testComm, return (aggFileSizeFromStat); } + + diff --git a/src/ior.c b/src/ior.c index e9d6c20..1ab0672 100644 --- a/src/ior.c +++ b/src/ior.c @@ -187,6 +187,7 @@ void init_IOR_Param_t(IOR_param_t * p) p->testComm = MPI_COMM_WORLD; p->setAlignment = 1; p->lustre_start_ost = -1; + p->lustre_use_all_osts = -1; } /* @@ -1460,7 +1461,7 @@ static void ShowTestInfo(IOR_param_t *params) */ static void ShowSetup(IOR_param_t *params) { - + int i; if (strcmp(params->debug, "") != 0) { printf("\n*** DEBUG MODE ***\n"); printf("*** %s ***\n\n", params->debug); @@ -1528,6 +1529,22 @@ static void ShowSetup(IOR_param_t *params) printf("\t stripe count = %d\n", params->lustre_stripe_count); } + if (params->lustre_use_all_osts == -1) { + printf("\t scatter to OSTs= false\n"); + } else { + if(params->lustre_use_all_osts == 0) { + printf("\t scatter to OSTs = all\n"); + } else { + printf("\t scatter to OSTs = %d\n",params->lustre_use_all_osts); + if(params->lustre_ost_list[0] >= 0) { + printf("\t OSTs list = [%d",params->lustre_ost_list[0]); + for(i=1;ilustre_use_all_osts;i++) { + printf(",%d",params->lustre_ost_list[i]); + } + printf("]"); + } + } + } } #endif /* HAVE_LUSTRE_LUSTRE_USER_H */ if (params->deadlineForStonewalling > 0) { diff --git a/src/ior.h b/src/ior.h index 60ae944..9e5f248 100644 --- a/src/ior.h +++ b/src/ior.h @@ -125,6 +125,8 @@ typedef struct int lustre_start_ost; int lustre_set_striping; /* flag that we need to set lustre striping */ int lustre_ignore_locks; + int lustre_use_all_osts; + int lustre_ost_list[MAX_OST_COUNT]; int id; /* test's unique ID */ int intraTestBarriers; /* barriers between open/op and op/close */ diff --git a/src/iordef.h b/src/iordef.h index 632d617..8c2d17e 100644 --- a/src/iordef.h +++ b/src/iordef.h @@ -108,6 +108,9 @@ extern int verbose; /* verbose output */ #ifndef PATH_MAX #define PATH_MAX 4096 #endif +#ifndef MAX_OST_COUNT +#define MAX_OST_COUNT 1024 +#endif #define DELIMITERS " \t\r\n=" /* ReadScript() */ #define FILENAME_DELIMITER '@' /* ParseFileName() */ diff --git a/src/parse_options.c b/src/parse_options.c index 452bcdb..f9c1ea1 100644 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -115,6 +115,7 @@ static void CheckRunSettings(IOR_test_t *tests) } } + /* * Set flags from commandline string/value pairs. */ @@ -263,6 +264,17 @@ void DecodeDirective(char *line, IOR_param_t *params) ERR("ior was not compiled with Lustre support"); #endif params->lustre_ignore_locks = atoi(value); + } else if (strcasecmp(option, "lustreuseallosts") == 0) { +#ifndef HAVE_LUSTRE_LUSTRE_USER_H + ERR("ior was not compiled with Lustre support"); +#ifndef HAVE_LUSTRE_LIBLUSTREAPI_H + ERR("ior was not compiled with Lustre API support"); +#endif + params->lustre_use_all_osts=-1; +#else + nodesete(value,strlen(value),&(params->lustre_use_all_osts),params->lustre_ost_list); + params->lustre_set_striping = 1; +#endif } else if (strcasecmp(option, "numtasks") == 0) { params->numTasks = atoi(value); RecalculateExpectedFileSize(params); @@ -278,20 +290,33 @@ void DecodeDirective(char *line, IOR_param_t *params) /* * Parse a single line, which may contain multiple comma-seperated directives + * Patch : don't consider comma inside brackets */ void ParseLine(char *line, IOR_param_t * test) { - char *start, *end; + char *start, *end, *endofline; + char *bracket; start = line; + endofline = start + strlen(start); do { end = strchr(start, ','); - if (end != NULL) + if (end != NULL) { *end = '\0'; + bracket = strchr(start,'['); + if (bracket != NULL) { + *end = ','; + end = strchr(start,']'); + end += 1; + *end = '\0'; + } + } DecodeDirective(start, test); - start = end + 1; + if(end < endofline) + start = end + 1; + else + end = NULL; } while (end != NULL); - } /* diff --git a/src/utilities.c b/src/utilities.c index 61b4f85..9ad34ec 100644 --- a/src/utilities.c +++ b/src/utilities.c @@ -430,3 +430,129 @@ int uname(struct utsname *name) return 0; } #endif /* _WIN32 */ + +/* + * Convert a string list into indexes + * ex: "[0,1,6-9,12,15-18]" gives the following array of integers [0,1,6,7,8,9,12,15,16,17,18] + */ +int nodesete(char *input, int input_len, int *count, int *list) { + int index=0; + char tmp1[1024]; + char tmp2[1024]; + char *ptmp; + int itmp; + int i; + int icount; + + icount = 0; + if(input[index] != '[') { + *count = atoi(input); + list[index]=-1; + return 0; + } else { + index++; + ptmp = tmp1; + itmp = 0; + while((input[index] != '\0') && (index < input_len)) { + if(input[index] == ',') { + if(ptmp == tmp2) { + ptmp[itmp] = '\0'; + for(i=atoi(tmp1);i<=atoi(tmp2);i++) { + list[icount]=i; + icount++; + } + ptmp = tmp1; + itmp = 0; + } else { + ptmp[itmp] = '\0'; + list[icount]=atoi(tmp1); + (icount)++; + itmp = 0; + } + index++; + } + if(input[index] == ']') { + if(ptmp == tmp2) { + ptmp[itmp] = '\0'; + for(i=atoi(tmp1);i<=atoi(tmp2);i++) { + list[icount]=i; + icount++; + } + ptmp = tmp1; + itmp = 0; + } else { + ptmp[itmp] = '\0'; + list[icount]=atoi(tmp1); + (icount)++; + itmp = 0; + } + break; + } + if(input[index] == '-') { + ptmp[itmp] = '\0'; + ptmp = tmp2; + itmp = 0; + index++; + } + ptmp[itmp] = input[index]; + index++; + itmp++; + } + *count = icount; + } + return 0; +} + +#ifdef HAVE_LUSTRE_LIBLUSTREAPI_H +#include +#include +#include +#ifndef PATH_MAX +#define PATH_MAX 4096 +#endif +int get_active_ost_list(const char* path, int *ost_count, int *ost_list) +{ + char mntdir[PATH_MAX] = ""; + char fsname[PATH_MAX] = ""; + char targetobd[PATH_MAX] = ""; + struct obd_uuid name; + char buf[1024]; + int fd; + FILE *fp; + int rc = 0; + int index; + char uuid[sizeof(struct obd_uuid)]; + + fd = open(path, O_RDONLY); + if(fd == 0) { + rc = -errno; + llapi_error(LLAPI_MSG_ERROR, rc, "error: opening '%s'", path); + return rc; + } + rc=llapi_file_fget_lov_uuid(fd, &name); + if (rc) + return rc; + close(fd); + + snprintf(targetobd,sizeof(targetobd),"/proc/fs/lustre/lov/%s/target_obd",name.uuid); + + fp = fopen(targetobd,"r"); + if (fp == NULL) { + rc = -errno; + llapi_error(LLAPI_MSG_ERROR, rc, "error: opening '%s'", targetobd); + return rc; + } + + *ost_count = 0; + while (fgets(buf,sizeof(buf), fp) != NULL) { + sscanf(buf, "%d: %s", &index, uuid); + if(ost_list != NULL) { + ost_list[*ost_count]=index; + } + (*ost_count)++; + } + fclose(fp); + return rc; +}; +#endif + diff --git a/src/utilities.h b/src/utilities.h index e3dcabc..9a7827b 100644 --- a/src/utilities.h +++ b/src/utilities.h @@ -27,5 +27,6 @@ void DumpBuffer(void *, size_t); void SeedRandGen(MPI_Comm); void SetHints (MPI_Info *, char *); void ShowHints (MPI_Info *); +int get_active_ost_list(const char* path, int *ost_count, int *ost_list); #endif /* !_UTILITIES_H */