Asynchronously stream multiple sources into an AWS S3 bucket

Condense data from multiple sources into a single file, and upload it to AWS S3 for later use.

Assuming appsettings.json contains a section with an array of URLs, the following code runs through each URL and streams the data into memory, before uploading it all asynchronously to an S3 bucket.

[Route("api/[controller]")]
[ApiController]
public class MyController(
    IOptions<List<string>> options,
    IAmazonS3 s3Client) : ControllerBase
{
    [HttpGet]
    public async Task<IActionResult> GetData(CancellationToken cancellationToken)
    {
        var urls = options.Value;

        if (urls is null)
        {
            throw new Exception("No data sources found");
        }

        var putRequest = new PutObjectRequest
        {
            BucketName = "my-bucket",
            Key = "data.json",
        };

        var httpClient = new HttpClient();

        using var memoryStream = new MemoryStream();

        foreach (var url in urls)
        {
            try
            {
                var dataResponse = await httpClient.GetStreamAsync(url, cancellationToken);
                await dataResponse.CopyToAsync(memoryStream, cancellationToken);
            }
            catch (Exception ex)
            {
                // Log error, but carry on with the rest of the URLs
                continue;
            }
        }

        putRequest.InputStream = memoryStream;
        var putResponse = await s3Client.PutObjectAsync(putRequest, cancellationToken);

        return StatusCode((int)putResponse.HttpStatusCode);
    }
}