fs_mgr/libfiemap/fiemap_writer_test.cpp

/*
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <fcntl.h>
#include <inttypes.h>
#include <linux/limits.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>

#include <string>
#include <utility>

#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/stringprintf.h>
#include <android-base/unique_fd.h>
#include <fstab/fstab.h>
#include <gtest/gtest.h>
#include <libdm/loop_control.h>
#include <libfiemap/fiemap_writer.h>
#include <libfiemap/split_fiemap_writer.h>
#include <libgsi/libgsi.h>
#include <storage_literals/storage_literals.h>

#include "utility.h"

namespace android {
namespace fiemap {

using namespace std;
using namespace std::string_literals;
using namespace android::fiemap;
using namespace android::storage_literals;
using unique_fd = android::base::unique_fd;
using LoopDevice = android::dm::LoopDevice;

std::string gTestDir;
uint64_t testfile_size = 536870912;  // default of 512MiB
size_t gBlockSize = 0;

class FiemapWriterTest : public ::testing::Test {
  protected:
    void SetUp() override {
        const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
        testfile = gTestDir + "/"s + tinfo->name();
    }

    void TearDown() override { unlink(testfile.c_str()); }

    // name of the file we use for testing
    std::string testfile;
};

class SplitFiemapTest : public ::testing::Test {
  protected:
    void SetUp() override {
        const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
        testfile = gTestDir + "/"s + tinfo->name();
    }

    void TearDown() override {
        std::string message;
        if (!SplitFiemap::RemoveSplitFiles(testfile, &message)) {
            cerr << "Could not remove all split files: " << message;
        }
    }

    // name of the file we use for testing
    std::string testfile;
};

TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) {
    // Try creating a file of size ~100TB but aligned to
    // 512 byte to make sure block alignment tests don't
    // fail.
    FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1099511627997184);
    EXPECT_EQ(fptr, nullptr);
    EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
    EXPECT_EQ(errno, ENOENT);
}

TEST_F(FiemapWriterTest, CreateUnalignedFile) {
    // Try creating a file of size 4097 bytes which is guaranteed
    // to be unaligned to all known block sizes.
    FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize + 1);
    ASSERT_NE(fptr, nullptr);
    ASSERT_EQ(fptr->size(), gBlockSize * 2);
}

TEST_F(FiemapWriterTest, CheckFilePath) {
    FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
    ASSERT_NE(fptr, nullptr);
    EXPECT_EQ(fptr->size(), gBlockSize);
    EXPECT_EQ(fptr->file_path(), testfile);
    EXPECT_EQ(access(testfile.c_str(), F_OK), 0);
}

TEST_F(FiemapWriterTest, CheckFileSize) {
    // Create a large-ish file and test that the expected size matches.
    FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1024 * 1024 * 16);
    ASSERT_NE(fptr, nullptr);

    struct stat s;
    ASSERT_EQ(stat(testfile.c_str(), &s), 0);
    EXPECT_EQ(static_cast<uint64_t>(s.st_size), fptr->size());
}

TEST_F(FiemapWriterTest, CheckProgress) {
    std::vector<uint64_t> expected;
    size_t invocations = 0;
    auto callback = [&](uint64_t done, uint64_t total) -> bool {
        if (invocations >= expected.size()) {
            return false;
        }
        EXPECT_EQ(done, expected[invocations]);
        EXPECT_EQ(total, gBlockSize);
        invocations++;
        return true;
    };

    expected.push_back(gBlockSize);

    auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
    EXPECT_NE(ptr, nullptr);
    EXPECT_EQ(invocations, expected.size());
}

TEST_F(FiemapWriterTest, CheckPinning) {
    auto ptr = FiemapWriter::Open(testfile, 4096);
    ASSERT_NE(ptr, nullptr);
    EXPECT_TRUE(FiemapWriter::HasPinnedExtents(testfile));
}

TEST_F(FiemapWriterTest, CheckBlockDevicePath) {
    FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
    EXPECT_EQ(fptr->size(), gBlockSize);
    EXPECT_EQ(fptr->bdev_path().find("/dev/block/"), size_t(0));

    if (!android::gsi::IsGsiRunning()) {
        EXPECT_EQ(fptr->bdev_path().find("/dev/block/dm-"), string::npos);
    }
}

TEST_F(FiemapWriterTest, CheckFileCreated) {
    FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 32768);
    ASSERT_NE(fptr, nullptr);
    unique_fd fd(open(testfile.c_str(), O_RDONLY));
    EXPECT_GT(fd, -1);
}

TEST_F(FiemapWriterTest, CheckFileSizeActual) {
    FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
    ASSERT_NE(fptr, nullptr);

    struct stat sb;
    ASSERT_EQ(stat(testfile.c_str(), &sb), 0);
    EXPECT_GE(sb.st_size, testfile_size);
}

TEST_F(FiemapWriterTest, CheckFileExtents) {
    FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
    ASSERT_NE(fptr, nullptr);
    EXPECT_GT(fptr->extents().size(), 0);
}

TEST_F(FiemapWriterTest, ExistingFile) {
    // Create the file.
    { ASSERT_NE(FiemapWriter::Open(testfile, gBlockSize), nullptr); }
    // Test that we can still open it.
    {
        auto ptr = FiemapWriter::Open(testfile, 0, false);
        ASSERT_NE(ptr, nullptr);
        EXPECT_GT(ptr->extents().size(), 0);
    }
}

TEST_F(FiemapWriterTest, FileDeletedOnError) {
    auto callback = [](uint64_t, uint64_t) -> bool { return false; };
    auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
    EXPECT_EQ(ptr, nullptr);
    EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
    EXPECT_EQ(errno, ENOENT);
}

TEST_F(FiemapWriterTest, MaxBlockSize) {
    uint64_t max_piece_size = 0;
    ASSERT_TRUE(DetermineMaximumFileSize(testfile, &max_piece_size));
    ASSERT_GT(max_piece_size, 0);
}

TEST_F(FiemapWriterTest, FibmapBlockAddressing) {
    FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
    ASSERT_NE(fptr, nullptr);

    switch (fptr->fs_type()) {
        case F2FS_SUPER_MAGIC:
        case EXT4_SUPER_MAGIC:
            // Skip the test for FIEMAP supported filesystems. This is really
            // because f2fs/ext4 have caches that seem to defeat reading back
            // directly from the block device, and writing directly is too
            // dangerous.
            std::cout << "Skipping test, filesystem does not use FIBMAP\n";
            return;
    }

    bool uses_dm;
    std::string bdev_path;
    ASSERT_TRUE(FiemapWriter::GetBlockDeviceForFile(testfile, &bdev_path, &uses_dm));

    if (uses_dm) {
        // We could use a device-mapper wrapper here to bypass encryption, but
        // really this test is for FIBMAP correctness on VFAT (where encryption
        // is never used), so we don't bother.
        std::cout << "Skipping test, block device is metadata encrypted\n";
        return;
    }

    std::string data(fptr->size(), '\0');
    for (size_t i = 0; i < data.size(); i++) {
        data[i] = 'A' + static_cast<char>(data.size() % 26);
    }

    {
        unique_fd fd(open(testfile.c_str(), O_WRONLY | O_CLOEXEC));
        ASSERT_GE(fd, 0);
        ASSERT_TRUE(android::base::WriteFully(fd, data.data(), data.size()));
        ASSERT_EQ(fsync(fd), 0);
    }

    ASSERT_FALSE(fptr->extents().empty());
    const auto& first_extent = fptr->extents()[0];

    unique_fd bdev(open(fptr->bdev_path().c_str(), O_RDONLY | O_CLOEXEC));
    ASSERT_GE(bdev, 0);

    off_t where = first_extent.fe_physical;
    ASSERT_EQ(lseek(bdev, where, SEEK_SET), where);

    // Note: this will fail on encrypted folders.
    std::string actual(data.size(), '\0');
    ASSERT_GE(first_extent.fe_length, data.size());
    ASSERT_TRUE(android::base::ReadFully(bdev, actual.data(), actual.size()));
    EXPECT_EQ(memcmp(actual.data(), data.data(), data.size()), 0);
}

TEST_F(FiemapWriterTest, CheckEmptyFile) {
    // Can't get any fiemap_extent out of a zero-sized file.
    FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 0);
    EXPECT_EQ(fptr, nullptr);
    EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
}

TEST_F(SplitFiemapTest, Create) {
    auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
    ASSERT_NE(ptr, nullptr);

    auto extents = ptr->extents();

    // Destroy the fiemap, closing file handles. This should not delete them.
    ptr = nullptr;

    std::vector<std::string> files;
    ASSERT_TRUE(SplitFiemap::GetSplitFileList(testfile, &files));
    for (const auto& path : files) {
        EXPECT_EQ(access(path.c_str(), F_OK), 0);
    }

    ASSERT_GE(extents.size(), files.size());
}

TEST_F(SplitFiemapTest, Open) {
    {
        auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
        ASSERT_NE(ptr, nullptr);
    }

    auto ptr = SplitFiemap::Open(testfile);
    ASSERT_NE(ptr, nullptr);

    auto extents = ptr->extents();
    ASSERT_GE(extents.size(), 24);
}

TEST_F(SplitFiemapTest, DeleteOnFail) {
    auto ptr = SplitFiemap::Create(testfile, 1024 * 1024 * 100, 1);
    ASSERT_EQ(ptr, nullptr);

    std::string first_file = testfile + ".0001";
    ASSERT_NE(access(first_file.c_str(), F_OK), 0);
    ASSERT_EQ(errno, ENOENT);
    ASSERT_NE(access(testfile.c_str(), F_OK), 0);
    ASSERT_EQ(errno, ENOENT);
}

TEST_F(SplitFiemapTest, CorruptSplit) {
    unique_fd fd(open(testfile.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0700));
    ASSERT_GE(fd, 0);

    // Make a giant random string.
    std::vector<char> data;
    for (size_t i = 0x1; i < 0x7f; i++) {
        for (size_t j = 0; j < 100; j++) {
            data.emplace_back(i);
        }
    }
    ASSERT_GT(data.size(), PATH_MAX);

    data.emplace_back('\n');

    ASSERT_TRUE(android::base::WriteFully(fd, data.data(), data.size()));
    fd = {};

    ASSERT_TRUE(SplitFiemap::RemoveSplitFiles(testfile));
}

static string ReadSplitFiles(const std::string& base_path, size_t num_files) {
    std::string result;
    for (int i = 0; i < num_files; i++) {
        std::string path = base_path + android::base::StringPrintf(".%04d", i);
        std::string data;
        if (!android::base::ReadFileToString(path, &data)) {
            return {};
        }
        result += data;
    }
    return result;
}

TEST_F(SplitFiemapTest, WriteWholeFile) {
    static constexpr size_t kChunkSize = 32768;
    static constexpr size_t kSize = kChunkSize * 3;
    auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
    ASSERT_NE(ptr, nullptr);

    auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
    for (size_t i = 0; i < kSize / sizeof(int); i++) {
        buffer[i] = i;
    }
    ASSERT_TRUE(ptr->Write(buffer.get(), kSize));

    std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
    auto actual = ReadSplitFiles(testfile, 3);
    ASSERT_EQ(expected.size(), actual.size());
    EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}

TEST_F(SplitFiemapTest, WriteFileInChunks1) {
    static constexpr size_t kChunkSize = 32768;
    static constexpr size_t kSize = kChunkSize * 3;
    auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
    ASSERT_NE(ptr, nullptr);

    auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
    for (size_t i = 0; i < kSize / sizeof(int); i++) {
        buffer[i] = i;
    }

    // Write in chunks of 1000 (so some writes straddle the boundary of two
    // files).
    size_t bytes_written = 0;
    while (bytes_written < kSize) {
        size_t to_write = std::min(kSize - bytes_written, (size_t)1000);
        char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
        ASSERT_TRUE(ptr->Write(data, to_write));
        bytes_written += to_write;
    }

    std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
    auto actual = ReadSplitFiles(testfile, 3);
    ASSERT_EQ(expected.size(), actual.size());
    EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}

TEST_F(SplitFiemapTest, WriteFileInChunks2) {
    static constexpr size_t kChunkSize = 32768;
    static constexpr size_t kSize = kChunkSize * 3;
    auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
    ASSERT_NE(ptr, nullptr);

    auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
    for (size_t i = 0; i < kSize / sizeof(int); i++) {
        buffer[i] = i;
    }

    // Write in chunks of 32KiB so every write is exactly at the end of the
    // current file.
    size_t bytes_written = 0;
    while (bytes_written < kSize) {
        size_t to_write = std::min(kSize - bytes_written, kChunkSize);
        char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
        ASSERT_TRUE(ptr->Write(data, to_write));
        bytes_written += to_write;
    }

    std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
    auto actual = ReadSplitFiles(testfile, 3);
    ASSERT_EQ(expected.size(), actual.size());
    EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}

TEST_F(SplitFiemapTest, WritePastEnd) {
    static constexpr size_t kChunkSize = 32768;
    static constexpr size_t kSize = kChunkSize * 3;
    auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
    ASSERT_NE(ptr, nullptr);

    auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
    for (size_t i = 0; i < kSize / sizeof(int); i++) {
        buffer[i] = i;
    }
    ASSERT_TRUE(ptr->Write(buffer.get(), kSize));
    ASSERT_FALSE(ptr->Write(buffer.get(), kSize));
}

// Get max file size and free space.
std::pair<uint64_t, uint64_t> GetBigFileLimit(const std::string& mount_point) {
    struct statvfs fs;
    if (statvfs(mount_point.c_str(), &fs) < 0) {
        PLOG(ERROR) << "statfs failed";
        return {0, 0};
    }

    auto fs_limit = static_cast<uint64_t>(fs.f_blocks) * (fs.f_bsize - 1);
    auto fs_free = static_cast<uint64_t>(fs.f_bfree) * fs.f_bsize;

    LOG(INFO) << "Big file limit: " << fs_limit << ", free space: " << fs_free;

    return {fs_limit, fs_free};
}

class FsTest : public ::testing::Test {
  protected:
    // 2GB Filesystem and 4k block size by default
    static constexpr uint64_t block_size = 4096;
    static constexpr uint64_t fs_size = 64 * 1024 * 1024;

    void SetUp() {
        android::fs_mgr::Fstab fstab;
        ASSERT_TRUE(android::fs_mgr::ReadFstabFromFile("/proc/mounts", &fstab));

        ASSERT_EQ(access(tmpdir_.path, F_OK), 0);
        fs_path_ = tmpdir_.path + "/fs_image"s;
        mntpoint_ = tmpdir_.path + "/mnt_point"s;

        auto entry = android::fs_mgr::GetEntryForMountPoint(&fstab, "/data");
        ASSERT_NE(entry, nullptr);
        if (entry->fs_type == "ext4") {
            SetUpExt4();
        } else if (entry->fs_type == "f2fs") {
            SetUpF2fs();
        } else {
            FAIL() << "Unrecognized fs_type: " << entry->fs_type;
        }
    }

    void SetUpExt4() {
        uint64_t count = fs_size / block_size;
        std::string dd_cmd =
                ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
                                              " count=%" PRIu64 " > /dev/null 2>&1",
                                              fs_path_.c_str(), block_size, count);
        std::string mkfs_cmd =
                ::android::base::StringPrintf("/system/bin/mkfs.ext4 -q %s", fs_path_.c_str());
        // create mount point
        ASSERT_EQ(mkdir(mntpoint_.c_str(), S_IRWXU), 0);
        // create file for the file system
        int ret = system(dd_cmd.c_str());
        ASSERT_EQ(ret, 0);
        // Get and attach a loop device to the filesystem we created
        LoopDevice loop_dev(fs_path_, 10s);
        ASSERT_TRUE(loop_dev.valid());
        // create file system
        ret = system(mkfs_cmd.c_str());
        ASSERT_EQ(ret, 0);

        // mount the file system
        ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint_.c_str(), "ext4", 0, nullptr), 0);
    }

    void SetUpF2fs() {
        uint64_t count = fs_size / block_size;
        std::string dd_cmd =
                ::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
                                              " count=%" PRIu64 " > /dev/null 2>&1",
                                              fs_path_.c_str(), block_size, count);
        std::string mkfs_cmd =
                ::android::base::StringPrintf("/system/bin/make_f2fs -q %s", fs_path_.c_str());
        // create mount point
        ASSERT_EQ(mkdir(mntpoint_.c_str(), S_IRWXU), 0);
        // create file for the file system
        int ret = system(dd_cmd.c_str());
        ASSERT_EQ(ret, 0);
        // Get and attach a loop device to the filesystem we created
        LoopDevice loop_dev(fs_path_, 10s);
        ASSERT_TRUE(loop_dev.valid());
        // create file system
        ret = system(mkfs_cmd.c_str());
        ASSERT_EQ(ret, 0);

        // mount the file system
        ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint_.c_str(), "f2fs", 0, nullptr), 0);
    }

    void TearDown() override {
        umount(mntpoint_.c_str());
        rmdir(mntpoint_.c_str());
        unlink(fs_path_.c_str());
    }

    TemporaryDir tmpdir_;
    std::string mntpoint_;
    std::string fs_path_;
};

TEST_F(FsTest, LowSpaceError) {
    auto limits = GetBigFileLimit(mntpoint_);
    ASSERT_GE(limits.first, 0);

    FiemapUniquePtr ptr;

    auto test_file = mntpoint_ + "/big_file";
    auto status = FiemapWriter::Open(test_file, limits.first, &ptr);
    ASSERT_FALSE(status.is_ok());
    ASSERT_EQ(status.error_code(), FiemapStatus::ErrorCode::NO_SPACE);

    // Also test for EFBIG.
    status = FiemapWriter::Open(test_file, 16_TiB, &ptr);
    ASSERT_FALSE(status.is_ok());
    ASSERT_NE(status.error_code(), FiemapStatus::ErrorCode::NO_SPACE);
}

bool DetermineBlockSize() {
    struct statfs s;
    if (statfs(gTestDir.c_str(), &s)) {
        std::cerr << "Could not call statfs: " << strerror(errno) << "\n";
        return false;
    }
    if (!s.f_bsize) {
        std::cerr << "Invalid block size: " << s.f_bsize << "\n";
        return false;
    }

    gBlockSize = s.f_bsize;
    return true;
}

}  // namespace fiemap
}  // namespace android

using namespace android::fiemap;

int main(int argc, char** argv) {
    ::testing::InitGoogleTest(&argc, argv);
    if (argc > 1 && argv[1] == "-h"s) {
        cerr << "Usage: [test_dir] [file_size]\n";
        cerr << "\n";
        cerr << "Note: test_dir must be a writable, unencrypted directory.\n";
        exit(EXIT_FAILURE);
    }
    ::android::base::InitLogging(argv, ::android::base::StderrLogger);

    std::string root_dir = "/data/local/unencrypted";
    if (access(root_dir.c_str(), F_OK)) {
        root_dir = "/data";
    }

    std::string tempdir = root_dir + "/XXXXXX"s;
    if (!mkdtemp(tempdir.data())) {
        cerr << "unable to create tempdir on " << root_dir << "\n";
        exit(EXIT_FAILURE);
    }
    if (!android::base::Realpath(tempdir, &gTestDir)) {
        cerr << "unable to find realpath for " << tempdir;
        exit(EXIT_FAILURE);
    }

    if (argc > 2) {
        testfile_size = strtoull(argv[2], NULL, 0);
        if (testfile_size == ULLONG_MAX) {
            testfile_size = 512 * 1024 * 1024;
        }
    }

    if (!DetermineBlockSize()) {
        exit(EXIT_FAILURE);
    }

    auto result = RUN_ALL_TESTS();

    std::string cmd = "rm -rf " + gTestDir;
    system(cmd.c_str());

    return result;
}