Test API to control a network throughput test tool

Overview

Generic high level test API to control a network throughput test tool. More…

// typedefs

typedef enum tapi_perf_bench tapi_perf_bench;
typedef enum tapi_perf_error tapi_perf_error;
typedef enum tapi_perf_report_kind tapi_perf_report_kind;
typedef struct tapi_perf_report tapi_perf_report;
typedef struct tapi_perf_server tapi_perf_server;
typedef struct tapi_perf_opts tapi_perf_opts;

typedef void (*tapi_perf_server_method_build_args)(
    te_vec *args,
    const tapi_perf_opts *options
    );

typedef te_errno (*tapi_perf_server_method_get_report)(
    tapi_perf_server *server,
    tapi_perf_report_kind kind,
    tapi_perf_report *report
    );

typedef struct tapi_perf_server_methods tapi_perf_server_methods;
typedef struct tapi_perf_client tapi_perf_client;

typedef void (*tapi_perf_client_method_build_args)(
    te_vec *args,
    const tapi_perf_opts *options
    );

typedef te_errno (*tapi_perf_client_method_wait)(
    tapi_perf_client *client,
    int16_t timeout
    );

typedef te_errno (*tapi_perf_client_method_get_report)(
    tapi_perf_client *client,
    tapi_perf_report_kind kind,
    tapi_perf_report *report
    );

typedef struct tapi_perf_client_methods tapi_perf_client_methods;
typedef struct tapi_perf_app tapi_perf_app;

// enums

enum tapi_perf_bench;
enum tapi_perf_error;
enum tapi_perf_report_kind;

// structs

struct tapi_perf_app;
struct tapi_perf_client;
struct tapi_perf_client_methods;
struct tapi_perf_opts;
struct tapi_perf_report;
struct tapi_perf_server;
struct tapi_perf_server_methods;

// global functions

void tapi_perf_opts_init(tapi_perf_opts* opts);
tapi_perf_server* tapi_perf_server_create(tapi_perf_bench bench, const tapi_perf_opts* options, tapi_job_factory_t* factory);
void tapi_perf_server_destroy(tapi_perf_server* server);
te_errno tapi_perf_server_start_unreliable(tapi_perf_server* server);
te_errno tapi_perf_server_start(tapi_perf_server* server);
te_errno tapi_perf_server_stop(tapi_perf_server* server);
te_errno tapi_perf_server_get_report(tapi_perf_server* server, tapi_perf_report* report);
te_errno tapi_perf_server_get_specific_report(tapi_perf_server* server, tapi_perf_report_kind kind, tapi_perf_report* report);
te_errno tapi_perf_server_check_report(tapi_perf_server* server, tapi_perf_report* report, const char* tag);
te_errno tapi_perf_server_get_check_report(tapi_perf_server* server, const char* tag, tapi_perf_report* report);
te_errno tapi_perf_server_get_dump_check_report(tapi_perf_server* server, const char* tag, tapi_perf_report* report);
tapi_perf_client* tapi_perf_client_create(tapi_perf_bench bench, const tapi_perf_opts* options, tapi_job_factory_t* factory);
void tapi_perf_client_destroy(tapi_perf_client* client);
te_errno tapi_perf_client_start(tapi_perf_client* client);
te_errno tapi_perf_client_stop(tapi_perf_client* client);
te_errno tapi_perf_client_wait(tapi_perf_client* client, int16_t timeout);
te_errno tapi_perf_client_get_report(tapi_perf_client* client, tapi_perf_report* report);
te_errno tapi_perf_client_get_specific_report(tapi_perf_client* client, tapi_perf_report_kind kind, tapi_perf_report* report);
te_errno tapi_perf_client_check_report(tapi_perf_client* client, tapi_perf_report* report, const char* tag);
te_errno tapi_perf_client_get_check_report(tapi_perf_client* client, const char* tag, tapi_perf_report* report);
te_errno tapi_perf_client_get_dump_check_report(tapi_perf_client* client, const char* tag, tapi_perf_report* report);
const char* tapi_perf_error2str(tapi_perf_error error);
const char* tapi_perf_bench2str(tapi_perf_bench bench);
static const char* tapi_perf_server_get_name(const tapi_perf_server* server);
static const char* tapi_perf_client_get_name(const tapi_perf_client* client);
void tapi_perf_log_report(const tapi_perf_server* server, const tapi_perf_client* client, const tapi_perf_report* report, const char* test_params);
void tapi_perf_log_cumulative_report(const tapi_perf_server* server[], const tapi_perf_client* client[], const tapi_perf_report* report[], int number_of_instances, const char* test_params);
bool tapi_perf_opts_cmp(const tapi_perf_opts* opts_a, const tapi_perf_opts* opts_b);

// macros

#define TAPI_PERF_BENCH_MAPPING_LIST
#define TAPI_PERF_INTERVAL_DISABLED
#define TAPI_PERF_TIMEOUT_DEFAULT
#define TEST_GET_PERF_BENCH(var_name_)

Detailed Documentation

Generic high level test API to control a network throughput test tool.

Copyright (C) 2004-2022 OKTET Labs Ltd. All rights reserved.

Notes

Throughput value should be obtained from receiver (usually it is a server instance until it is changed by traffic direction options)

You have to restart a server along with a client if you need to perform a few measurements in a row, otherwise the server returns a report of the first measurement all the time (until restarting or destroying)

Example of usage

Lets assume we need to send UDP traffic with iperf. We need to check the result throughput with the following input options: total bandwidth = 1000 Mbit/s, streams = 5, and test duration = 60 sec. Server iperf should use any free port on the host, and its host address is 192.168.1.1.

So, we have the following commands for both server and client:

iperf -s -u -p 60000

iperf -c 192.168.1.1 -p 60000 -u -b 200 -P 5 -t 60

#include "tapi_job.h"
#include "tapi_job_factory_rpc.h"
#include "tapi_performance.h"

int main(int argc, char *argv[])
{
    tapi_perf_server    *perf_server = NULL;
    tapi_perf_client    *perf_client = NULL;
    tapi_perf_opts       perf_opts;
    tapi_perf_report     perf_server_report;
    tapi_perf_report     perf_client_report;
    rcf_rpc_server      *perf_server_rpcs;
    rcf_rpc_server      *perf_client_rpcs;
    tapi_job_factory_t  *client_factory = NULL;
    tapi_job_factory_t  *server_factory = NULL;

    TEST_START;

    // Set default perf options
    tapi_perf_opts_init(&perf_opts);

    // Set test specific perf options
    perf_opts.host = "192.168.1.1";
    perf_opts.protocol = IPPROTO_UDP;
    perf_opts.port = tapi_get_port(perf_server_rpcs);
    perf_opts.streams = 5;
    perf_opts.bandwidth_bits = TE_UNITS_DEC_M2U(1000) / perf_opts.streams;
    perf_opts.duration_sec = 60;
    // To force server to print a report at the end of test even if it lost
    // connection with client (iperf tool issue, Bug 9714)
    perf_opts.interval_sec = perf_opts.duration_sec;

    CHECK_RC(tapi_job_factory_rpc_create(perf_server_rpcs, &server_factory));
    CHECK_RC(tapi_job_factory_rpc_create(perf_client_rpcs, &client_factory));
    perf_server = tapi_perf_server_create(TAPI_PERF_IPERF, &perf_opts,
                                          server_factory);
    perf_client = tapi_perf_client_create(TAPI_PERF_IPERF, &perf_opts,
                                          client_factory);
    CHECK_RC(tapi_perf_server_start(perf_server));
    CHECK_RC(tapi_perf_client_start(perf_client));
    CHECK_RC(tapi_perf_client_wait(perf_client, TAPI_PERF_TIMEOUT_DEFAULT));
    // Time is relative and goes differently on different hosts.
    // Sometimes we need to wait for a few moments until report is ready.
    VSLEEP(1, "ensure perf server has printed its report");

    CHECK_RC(tapi_perf_client_get_dump_check_report(perf_client, "client",
                                                    &perf_client_report));
    CHECK_RC(tapi_perf_server_get_dump_check_report(perf_server, "server",
                                                    &perf_server_report));

    // Analyze the obtained reports
    ...

    TEST_SUCCESS;

cleanup:
    tapi_perf_client_destroy(perf_client);
    tapi_perf_server_destroy(perf_server);
    tapi_job_factory_destroy(client_factory);
    tapi_job_factory_destroy(server_factory);
    TEST_END;
}

Typedefs

typedef enum tapi_perf_bench tapi_perf_bench

Supported network throughput test tools list.

typedef enum tapi_perf_error tapi_perf_error

List of possible network throughput test tool errors.

typedef enum tapi_perf_report_kind tapi_perf_report_kind

List of possible report kinds.

typedef struct tapi_perf_report tapi_perf_report

Network throughput test tool report.

typedef struct tapi_perf_server tapi_perf_server

Network throughput test server tool context

typedef struct tapi_perf_opts tapi_perf_opts

Network throughput test tool options

typedef void (*tapi_perf_server_method_build_args)(
    te_vec *args,
    const tapi_perf_opts *options
    )

Build command string to run server tool.

Parameters:

args	List of built commands line arguments.
options	Tool server options.

typedef te_errno (*tapi_perf_server_method_get_report)(
    tapi_perf_server *server,
    tapi_perf_report_kind kind,
    tapi_perf_report *report
    )

Get server report. The function reads client output (stdout, stderr).

Parameters:

server	Server context.
report	Report with results.

Returns:

Status code.

typedef struct tapi_perf_server_methods tapi_perf_server_methods

Methods to operate the server network throughput test tool.

typedef struct tapi_perf_client tapi_perf_client

Network throughput test client tool context

typedef void (*tapi_perf_client_method_build_args)(
    te_vec *args,
    const tapi_perf_opts *options
    )

Build command string to run client tool.

Parameters:

args	List of built commands line arguments.
options	Tool client options.

typedef te_errno (*tapi_perf_client_method_wait)(
    tapi_perf_client *client,
    int16_t timeout
    )

Wait while client finishes his work. Note, function jumps to cleanup if timeout is expired.

Parameters:

client	Client context.
timeout	Time to wait for client results (seconds). It MUST be big enough to finish client normally (it depends on client’s options), otherwise the function will be failed. Use TAPI_PERF_TIMEOUT_DEFAULT to coerce the function to calculate the required timeout value.

Returns:

Status code.

typedef te_errno (*tapi_perf_client_method_get_report)(
    tapi_perf_client *client,
    tapi_perf_report_kind kind,
    tapi_perf_report *report
    )

Get client report. The function reads client output (stdout, stderr).

Parameters:

client	Client context.
report	Report with results.

Returns:

Status code.

typedef struct tapi_perf_client_methods tapi_perf_client_methods

Methods to operate the client network throughput test tool.

typedef struct tapi_perf_app tapi_perf_app

Network throughput test tool context (common for both server and client)

Global Functions

void tapi_perf_opts_init(tapi_perf_opts* opts)

Initialize options with default values (from point of view of perf tool).

Parameters:

opts	Network throughput test tool options.

tapi_perf_server* tapi_perf_server_create(tapi_perf_bench bench, const tapi_perf_opts* options, tapi_job_factory_t* factory)

Create server network throughput test tool proxy.

Parameters:

bench	Sort of tool, see tapi_perf_bench to get a list of supported tools.
options	Server tool specific options, may be NULL, to set them to default, further you can edit them using return value.

Returns:

Status code.

See also:

tapi_perf_server_destroy

void tapi_perf_server_destroy(tapi_perf_server* server)

Destroy server network throughput test tool proxy.

Parameters:

server

Server context.

See also:

tapi_perf_server_create

te_errno tapi_perf_server_start_unreliable(tapi_perf_server* server)

Start perf server. It returns immediately after run the command starting the server. It can be unreliable to call tapi_perf_client_start() just after this function because server can not be ready to accept clients by this time, especially on slow machine. It is recommended to use this function only if there will be some delay before starting a client, otherwise use tapi_perf_server_start() instead.

Parameters:

server

Server context.

Returns:

Status code.

See also:

tapi_perf_server_start, tapi_perf_server_stop

te_errno tapi_perf_server_start(tapi_perf_server* server)

Start perf server “reliably”. It calls tapi_perf_server_start_unreliable() and wait until it is ready to accept clients. Note, it is not true reliable because it doesn’t check whether server is ready, or not, it just waits for some time.

Parameters:

server

Server context.

Returns:

Status code.

See also:

tapi_perf_server_start_unreliable, tapi_perf_server_stop

te_errno tapi_perf_server_stop(tapi_perf_server* server)

Stop perf server.

Parameters:

server

Server context.

Returns:

Status code.

See also:

tapi_perf_server_start

te_errno tapi_perf_server_get_report(tapi_perf_server* server, tapi_perf_report* report)

Get server report. The function reads server output (stdout, stderr).

Parameters:

server	Server context.
report	Report with results.

Returns:

Status code.

te_errno tapi_perf_server_get_specific_report(tapi_perf_server* server, tapi_perf_report_kind kind, tapi_perf_report* report)

Get server report of specified kind. The function reads server output (stdout, stderr).

Parameters:

server	Server context.
kind	Report kind, e.g. default or receiver’s, or sender’s.
report	Report with results.

Returns:

Status code.

te_errno tapi_perf_server_check_report(tapi_perf_server* server, tapi_perf_report* report, const char* tag)

Check server report for errors. The function prints verdicts in case of errors are presents in the report.

Parameters:

server	Server context.
report	Server report.
tag	Tag to print in verdict message.

Returns:

Status code. It returns non-zero code if there are errors in the report.

te_errno tapi_perf_server_get_check_report(tapi_perf_server* server, const char* tag, tapi_perf_report* report)

Get server report and check it for errors. The function is a wrapper which calls tapi_perf_server_get_report() and tapi_perf_server_check_report()

Parameters:

server	Server context.
tag	Tag to print in verdict message.
report	Report with results, it may be NULL if you don’t care about results, but only errors.

Returns:

Status code.

See also:

tapi_perf_server_get_report, tapi_perf_server_check_report, tapi_perf_server_get_dump_check_report

te_errno tapi_perf_server_get_dump_check_report(tapi_perf_server* server, const char* tag, tapi_perf_report* report)

Get server report, dump it to log and check for errors.

Parameters:

server	Server context.
tag	Tag to print in both verdict and dump messages.
report	Report with results, it may be NULL if you don’t care about results, but only errors.

Returns:

Status code.

See also:

tapi_perf_server_get_report, tapi_perf_server_check_report, tapi_perf_server_get_check_report

tapi_perf_client* tapi_perf_client_create(tapi_perf_bench bench, const tapi_perf_opts* options, tapi_job_factory_t* factory)

Create client network throughput test tool proxy.

Parameters:

bench	Sort of tool, see tapi_perf_bench to get a list of supported tools.
options	Client tool specific options, may be NULL, to set them to default, further you can edit them using return value.

Returns:

Status code.

See also:

tapi_perf_client_destroy

void tapi_perf_client_destroy(tapi_perf_client* client)

Destroy client network throughput test tool proxy.

Parameters:

client

Client context.

See also:

tapi_perf_client_create

te_errno tapi_perf_client_start(tapi_perf_client* client)

Start perf client.

Parameters:

client

Client context.

Returns:

Status code.

See also:

tapi_perf_client_stop

te_errno tapi_perf_client_stop(tapi_perf_client* client)

Stop perf client.

Parameters:

client

Client context.

Returns:

Status code.

See also:

tapi_perf_client_start

te_errno tapi_perf_client_wait(tapi_perf_client* client, int16_t timeout)

Wait while client finishes his work. Note, function jumps to cleanup if timeout is expired.

Parameters:

client	Client context.
timeout	Time to wait for client results (seconds). It MUST be big enough to finish client normally (it depends on client’s options), otherwise the function will be failed. Use TAPI_PERF_TIMEOUT_DEFAULT to coerce the function to calculate the required timeout value.

Returns:

Status code.

te_errno tapi_perf_client_get_report(tapi_perf_client* client, tapi_perf_report* report)

Get client report. The function reads client output (stdout, stderr).

Parameters:

client	Client context.
report	Report with results.

Returns:

Status code.

te_errno tapi_perf_client_get_specific_report(tapi_perf_client* client, tapi_perf_report_kind kind, tapi_perf_report* report)

Get client report of specified kind. The function reads client output (stdout, stderr).

Parameters:

client	Client context.
kind	Report kind, e.g. default or receiver’s, or sender’s.
report	Report with results.

Returns:

Status code.

te_errno tapi_perf_client_check_report(tapi_perf_client* client, tapi_perf_report* report, const char* tag)

Check client report for errors. The function prints verdicts in case of errors are presents in the report.

Parameters:

client	Client context.
report	Client report.
tag	Tag to print in verdict message.

Returns:

Status code. It returns non-zero code if there are errors in the report.

te_errno tapi_perf_client_get_check_report(tapi_perf_client* client, const char* tag, tapi_perf_report* report)

Get client report and check it for errors. The function is a wrapper which calls tapi_perf_client_get_report() and tapi_perf_client_check_report()

Parameters:

client	Client context.
tag	Tag to print in verdict message.
report	Report with results, it may be NULL if you don’t care about results, but only errors.

Returns:

Status code.

See also:

tapi_perf_client_get_report, tapi_perf_client_check_report, tapi_perf_client_get_dump_check_report

te_errno tapi_perf_client_get_dump_check_report(tapi_perf_client* client, const char* tag, tapi_perf_report* report)

Get client report, dump it to log and check for errors.

Parameters:

client	Client context.
tag	Tag to print in both verdict and dump messages.
report	Report with results, it may be NULL if you don’t care about results, but only errors.

Returns:

Status code.

See also:

tapi_perf_client_get_report, tapi_perf_client_check_report, tapi_perf_client_get_check_report

const char* tapi_perf_error2str(tapi_perf_error error)

Get error description.

Parameters:

error

Error code.

Returns:

Error code string representation.

const char* tapi_perf_bench2str(tapi_perf_bench bench)

Get string representation of bench.

Parameters:

bench

Tool’s sort.

Returns:

Tool’s sort name.

static const char* tapi_perf_server_get_name(const tapi_perf_server* server)

Get server network throughput test tool name.

Parameters:

server

Server context.

Returns:

Server tool name.

static const char* tapi_perf_client_get_name(const tapi_perf_client* client)

Get client network throughput test tool name.

Parameters:

client

Client context.

Returns:

Client tool name.

void tapi_perf_log_report(const tapi_perf_server* server, const tapi_perf_client* client, const tapi_perf_report* report, const char* test_params)

Print a network throughput test tool report.

Parameters:

server	Server context.
client	Client context.
report	Report.
test_params	Test specific params; It should be represented in the form of comma-separated pairs “param=value”.

void tapi_perf_log_cumulative_report(const tapi_perf_server* server[], const tapi_perf_client* client[], const tapi_perf_report* report[], int number_of_instances, const char* test_params)

Print a network throughput test tool report by adding throughput of all server/client pairs. Note, that we expect server/client pairs to run roughly the same traffic, see perf_opts_cmp() for details.

Parameters:

server	List of server contexts.
client	List of client contexts.
report	List of reports (user decides which one is taken where).
number_of_instances	Number of instances in the above 3 lists
test_params	Test specific params; It should be represented in the form of comma-separated pairs “param=value”.

bool tapi_perf_opts_cmp(const tapi_perf_opts* opts_a, const tapi_perf_opts* opts_b)

Compare important parts of the run.

Parameters:

opts_a	First object for comparison
opts_b	Second object for comparison

Returns:

true if objects’ important properties are equal, false if not

Macros

#define TAPI_PERF_BENCH_MAPPING_LIST

The list of values allowed for parameter of type ‘tapi_perf_bench’

#define TAPI_PERF_INTERVAL_DISABLED

Disable periodic bandwidth reports.

#define TAPI_PERF_TIMEOUT_DEFAULT

Default timeout to client wait method. It means the real timeout will be calculated according to tool’s options.

#define TEST_GET_PERF_BENCH(var_name_)

Get the value of parameter of type ‘tapi_perf_bench’

Parameters:

var_name_

Name of the variable used to get the value of “var_name_” parameter of type ‘tapi_perf_bench’ (OUT)