1  /*
2   * Copyright (C) 2019 The Android Open Source Project
3   *
4   * Licensed under the Apache License, Version 2.0 (the "License");
5   * you may not use this file except in compliance with the License.
6   * You may obtain a copy of the License at
7   *
8   *      http://www.apache.org/licenses/LICENSE-2.0
9   *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  
17  #include <android-base/file.h>
18  #include <android-base/logging.h>
19  #include <android-base/unique_fd.h>
20  #include <binder/ParcelFileDescriptor.h>
21  #include <gmock/gmock.h>
22  #include <gtest/gtest.h>
23  #include <utils/Log.h>
24  #include <utils/String16.h>
25  
26  #include <chrono>
27  #include <future>
28  
29  #include "IncrementalService.h"
30  #include "IncrementalServiceValidation.h"
31  #include "Metadata.pb.h"
32  #include "ServiceWrappers.h"
33  
34  using namespace testing;
35  using namespace android::incremental;
36  using namespace std::literals;
37  using testing::_;
38  using testing::Invoke;
39  using testing::NiceMock;
40  
41  #undef LOG_TAG
42  #define LOG_TAG "IncrementalServiceTest"
43  
44  using namespace android::incfs;
45  using namespace android::content::pm;
46  using PerUidReadTimeouts = android::os::incremental::PerUidReadTimeouts;
47  
48  namespace android::os::incremental {
49  
50  class MockVoldService : public VoldServiceWrapper {
51  public:
52      MOCK_CONST_METHOD5(mountIncFs,
53                         binder::Status(const std::string& backingPath, const std::string& targetDir,
54                                        int32_t flags, const std::string& sysfsName,
55                                        IncrementalFileSystemControlParcel* _aidl_return));
56      MOCK_CONST_METHOD1(unmountIncFs, binder::Status(const std::string& dir));
57      MOCK_CONST_METHOD2(bindMount,
58                         binder::Status(const std::string& sourceDir, const std::string& argetDir));
59      MOCK_CONST_METHOD4(
60              setIncFsMountOptions,
61              binder::Status(const ::android::os::incremental::IncrementalFileSystemControlParcel&,
62                             bool, bool, const std::string&));
63  
mountIncFsFails()64      void mountIncFsFails() {
65          ON_CALL(*this, mountIncFs(_, _, _, _, _))
66                  .WillByDefault(
67                          Return(binder::Status::fromExceptionCode(1, String8("failed to mount"))));
68      }
mountIncFsInvalidControlParcel()69      void mountIncFsInvalidControlParcel() {
70          ON_CALL(*this, mountIncFs(_, _, _, _, _))
71                  .WillByDefault(Invoke(this, &MockVoldService::getInvalidControlParcel));
72      }
mountIncFsSuccess()73      void mountIncFsSuccess() {
74          ON_CALL(*this, mountIncFs(_, _, _, _, _))
75                  .WillByDefault(Invoke(this, &MockVoldService::incFsSuccess));
76      }
bindMountFails()77      void bindMountFails() {
78          ON_CALL(*this, bindMount(_, _))
79                  .WillByDefault(Return(
80                          binder::Status::fromExceptionCode(1, String8("failed to bind-mount"))));
81      }
bindMountSuccess()82      void bindMountSuccess() {
83          ON_CALL(*this, bindMount(_, _)).WillByDefault(Return(binder::Status::ok()));
84      }
setIncFsMountOptionsFails() const85      void setIncFsMountOptionsFails() const {
86          ON_CALL(*this, setIncFsMountOptions(_, _, _, _))
87                  .WillByDefault(Return(
88                          binder::Status::fromExceptionCode(1, String8("failed to set options"))));
89      }
setIncFsMountOptionsSuccess()90      void setIncFsMountOptionsSuccess() {
91          ON_CALL(*this, setIncFsMountOptions(_, _, _, _))
92                  .WillByDefault(Invoke(this, &MockVoldService::setIncFsMountOptionsOk));
93      }
getInvalidControlParcel(const std::string & imagePath,const std::string & targetDir,int32_t flags,const std::string & sysfsName,IncrementalFileSystemControlParcel * _aidl_return)94      binder::Status getInvalidControlParcel(const std::string& imagePath,
95                                             const std::string& targetDir, int32_t flags,
96                                             const std::string& sysfsName,
97                                             IncrementalFileSystemControlParcel* _aidl_return) {
98          _aidl_return = {};
99          return binder::Status::ok();
100      }
incFsSuccess(const std::string & imagePath,const std::string & targetDir,int32_t flags,const std::string & sysfsName,IncrementalFileSystemControlParcel * _aidl_return)101      binder::Status incFsSuccess(const std::string& imagePath, const std::string& targetDir,
102                                  int32_t flags, const std::string& sysfsName,
103                                  IncrementalFileSystemControlParcel* _aidl_return) {
104          _aidl_return->pendingReads.reset(base::unique_fd(dup(STDIN_FILENO)));
105          _aidl_return->cmd.reset(base::unique_fd(dup(STDIN_FILENO)));
106          _aidl_return->log.reset(base::unique_fd(dup(STDIN_FILENO)));
107          return binder::Status::ok();
108      }
setIncFsMountOptionsOk(const::android::os::incremental::IncrementalFileSystemControlParcel & control,bool enableReadLogs,bool enableReadTimeouts,const std::string & sysfsName)109      binder::Status setIncFsMountOptionsOk(
110              const ::android::os::incremental::IncrementalFileSystemControlParcel& control,
111              bool enableReadLogs, bool enableReadTimeouts, const std::string& sysfsName) {
112          mReadLogsEnabled = enableReadLogs;
113          mReadTimeoutsEnabled = enableReadTimeouts;
114          return binder::Status::ok();
115      }
116  
readLogsEnabled() const117      bool readLogsEnabled() const { return mReadLogsEnabled; }
readTimeoutsEnabled() const118      bool readTimeoutsEnabled() const { return mReadTimeoutsEnabled; }
119  
120  private:
121      TemporaryFile cmdFile;
122      TemporaryFile logFile;
123  
124      bool mReadLogsEnabled = false;
125      bool mReadTimeoutsEnabled = true;
126  };
127  
128  class MockDataLoader : public IDataLoader {
129  public:
MockDataLoader()130      MockDataLoader() {
131          initializeCreateOk();
132          ON_CALL(*this, start(_)).WillByDefault(Invoke(this, &MockDataLoader::startOk));
133          ON_CALL(*this, stop(_)).WillByDefault(Invoke(this, &MockDataLoader::stopOk));
134          ON_CALL(*this, destroy(_)).WillByDefault(Invoke(this, &MockDataLoader::destroyOk));
135          ON_CALL(*this, prepareImage(_, _, _))
136                  .WillByDefault(Invoke(this, &MockDataLoader::prepareImageOk));
137      }
onAsBinder()138      IBinder* onAsBinder() override { return nullptr; }
139      MOCK_METHOD4(create,
140                   binder::Status(int32_t id, const DataLoaderParamsParcel& params,
141                                  const FileSystemControlParcel& control,
142                                  const sp<IDataLoaderStatusListener>& listener));
143      MOCK_METHOD1(start, binder::Status(int32_t id));
144      MOCK_METHOD1(stop, binder::Status(int32_t id));
145      MOCK_METHOD1(destroy, binder::Status(int32_t id));
146      MOCK_METHOD3(prepareImage,
147                   binder::Status(int32_t id, const std::vector<InstallationFileParcel>& addedFiles,
148                                  const std::vector<std::string>& removedFiles));
149  
initializeCreateOk()150      void initializeCreateOk() {
151          ON_CALL(*this, create(_, _, _, _)).WillByDefault(Invoke(this, &MockDataLoader::createOk));
152      }
153  
initializeCreateOkNoStatus()154      void initializeCreateOkNoStatus() {
155          ON_CALL(*this, create(_, _, _, _))
156                  .WillByDefault(Invoke(this, &MockDataLoader::createOkNoStatus));
157      }
158  
createOk(int32_t id,const content::pm::DataLoaderParamsParcel & params,const content::pm::FileSystemControlParcel & control,const sp<content::pm::IDataLoaderStatusListener> & listener)159      binder::Status createOk(int32_t id, const content::pm::DataLoaderParamsParcel& params,
160                              const content::pm::FileSystemControlParcel& control,
161                              const sp<content::pm::IDataLoaderStatusListener>& listener) {
162          createOkNoStatus(id, params, control, listener);
163          reportStatus(id);
164          return binder::Status::ok();
165      }
createOkNoStatus(int32_t id,const content::pm::DataLoaderParamsParcel & params,const content::pm::FileSystemControlParcel & control,const sp<content::pm::IDataLoaderStatusListener> & listener)166      binder::Status createOkNoStatus(int32_t id, const content::pm::DataLoaderParamsParcel& params,
167                                      const content::pm::FileSystemControlParcel& control,
168                                      const sp<content::pm::IDataLoaderStatusListener>& listener) {
169          mServiceConnector = control.service;
170          mListener = listener;
171          mStatus = IDataLoaderStatusListener::DATA_LOADER_CREATED;
172          return binder::Status::ok();
173      }
startOk(int32_t id)174      binder::Status startOk(int32_t id) {
175          setAndReportStatus(id, IDataLoaderStatusListener::DATA_LOADER_STARTED);
176          return binder::Status::ok();
177      }
stopOk(int32_t id)178      binder::Status stopOk(int32_t id) {
179          setAndReportStatus(id, IDataLoaderStatusListener::DATA_LOADER_STOPPED);
180          return binder::Status::ok();
181      }
destroyOk(int32_t id)182      binder::Status destroyOk(int32_t id) {
183          setAndReportStatus(id, IDataLoaderStatusListener::DATA_LOADER_DESTROYED);
184          mListener = nullptr;
185          return binder::Status::ok();
186      }
prepareImageOk(int32_t id,const::std::vector<content::pm::InstallationFileParcel> &,const::std::vector<::std::string> &)187      binder::Status prepareImageOk(int32_t id,
188                                    const ::std::vector<content::pm::InstallationFileParcel>&,
189                                    const ::std::vector<::std::string>&) {
190          setAndReportStatus(id, IDataLoaderStatusListener::DATA_LOADER_IMAGE_READY);
191          return binder::Status::ok();
192      }
setStorageParams(bool enableReadLogs)193      int32_t setStorageParams(bool enableReadLogs) {
194          int32_t result = -1;
195          EXPECT_NE(mServiceConnector.get(), nullptr);
196          EXPECT_TRUE(mServiceConnector->setStorageParams(enableReadLogs, &result).isOk());
197          return result;
198      }
status() const199      int status() const { return mStatus; }
200  
201  private:
setAndReportStatus(int id,int status)202      void setAndReportStatus(int id, int status) {
203          mStatus = status;
204          reportStatus(id);
205      }
reportStatus(int id)206      void reportStatus(int id) {
207          if (mListener) {
208              mListener->onStatusChanged(id, mStatus);
209          }
210      }
211  
212      sp<IIncrementalServiceConnector> mServiceConnector;
213      sp<IDataLoaderStatusListener> mListener;
214      int mStatus = IDataLoaderStatusListener::DATA_LOADER_DESTROYED;
215  };
216  
217  class MockDataLoaderManager : public DataLoaderManagerWrapper {
218  public:
MockDataLoaderManager(sp<IDataLoader> dataLoader)219      MockDataLoaderManager(sp<IDataLoader> dataLoader) : mDataLoaderHolder(std::move(dataLoader)) {
220          EXPECT_TRUE(mDataLoaderHolder != nullptr);
221      }
222  
223      MOCK_CONST_METHOD5(bindToDataLoader,
224                         binder::Status(int32_t mountId, const DataLoaderParamsParcel& params,
225                                        int bindDelayMs,
226                                        const sp<IDataLoaderStatusListener>& listener,
227                                        bool* _aidl_return));
228      MOCK_CONST_METHOD2(getDataLoader,
229                         binder::Status(int32_t mountId, sp<IDataLoader>* _aidl_return));
230      MOCK_CONST_METHOD1(unbindFromDataLoader, binder::Status(int32_t mountId));
231  
bindToDataLoaderSuccess()232      void bindToDataLoaderSuccess() {
233          ON_CALL(*this, bindToDataLoader(_, _, _, _, _))
234                  .WillByDefault(Invoke(this, &MockDataLoaderManager::bindToDataLoaderOk));
235      }
bindToDataLoaderFails()236      void bindToDataLoaderFails() {
237          ON_CALL(*this, bindToDataLoader(_, _, _, _, _))
238                  .WillByDefault(Return(
239                          (binder::Status::fromExceptionCode(1, String8("failed to prepare")))));
240      }
getDataLoaderSuccess()241      void getDataLoaderSuccess() {
242          ON_CALL(*this, getDataLoader(_, _))
243                  .WillByDefault(Invoke(this, &MockDataLoaderManager::getDataLoaderOk));
244      }
unbindFromDataLoaderSuccess()245      void unbindFromDataLoaderSuccess() {
246          ON_CALL(*this, unbindFromDataLoader(_))
247                  .WillByDefault(Invoke(this, &MockDataLoaderManager::unbindFromDataLoaderOk));
248      }
bindToDataLoaderOk(int32_t mountId,const DataLoaderParamsParcel & params,int bindDelayMs,const sp<IDataLoaderStatusListener> & listener,bool * _aidl_return)249      binder::Status bindToDataLoaderOk(int32_t mountId, const DataLoaderParamsParcel& params,
250                                        int bindDelayMs,
251                                        const sp<IDataLoaderStatusListener>& listener,
252                                        bool* _aidl_return) {
253          mId = mountId;
254          mListener = listener;
255          mDataLoader = mDataLoaderHolder;
256          mBindDelayMs = bindDelayMs;
257          *_aidl_return = true;
258          if (mListener) {
259              mListener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_BOUND);
260          }
261          return binder::Status::ok();
262      }
bindToDataLoaderNotOkWithNoDelay(int32_t mountId,const DataLoaderParamsParcel & params,int bindDelayMs,const sp<IDataLoaderStatusListener> & listener,bool * _aidl_return)263      binder::Status bindToDataLoaderNotOkWithNoDelay(int32_t mountId,
264                                                      const DataLoaderParamsParcel& params,
265                                                      int bindDelayMs,
266                                                      const sp<IDataLoaderStatusListener>& listener,
267                                                      bool* _aidl_return) {
268          CHECK(bindDelayMs == 0) << bindDelayMs;
269          *_aidl_return = false;
270          return binder::Status::ok();
271      }
bindToDataLoaderBindingWithNoDelay(int32_t mountId,const DataLoaderParamsParcel & params,int bindDelayMs,const sp<IDataLoaderStatusListener> & listener,bool * _aidl_return)272      binder::Status bindToDataLoaderBindingWithNoDelay(int32_t mountId,
273                                                        const DataLoaderParamsParcel& params,
274                                                        int bindDelayMs,
275                                                        const sp<IDataLoaderStatusListener>& listener,
276                                                        bool* _aidl_return) {
277          CHECK(bindDelayMs == 0) << bindDelayMs;
278          *_aidl_return = true;
279          if (listener) {
280              listener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_BINDING);
281          }
282          return binder::Status::ok();
283      }
bindToDataLoaderOkWithNoDelay(int32_t mountId,const DataLoaderParamsParcel & params,int bindDelayMs,const sp<IDataLoaderStatusListener> & listener,bool * _aidl_return)284      binder::Status bindToDataLoaderOkWithNoDelay(int32_t mountId,
285                                                   const DataLoaderParamsParcel& params,
286                                                   int bindDelayMs,
287                                                   const sp<IDataLoaderStatusListener>& listener,
288                                                   bool* _aidl_return) {
289          CHECK(bindDelayMs == 0) << bindDelayMs;
290          return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
291      }
bindToDataLoaderOkWith1sDelay(int32_t mountId,const DataLoaderParamsParcel & params,int bindDelayMs,const sp<IDataLoaderStatusListener> & listener,bool * _aidl_return)292      binder::Status bindToDataLoaderOkWith1sDelay(int32_t mountId,
293                                                   const DataLoaderParamsParcel& params,
294                                                   int bindDelayMs,
295                                                   const sp<IDataLoaderStatusListener>& listener,
296                                                   bool* _aidl_return) {
297          CHECK(100 * 9 <= bindDelayMs && bindDelayMs <= 100 * 11) << bindDelayMs;
298          return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
299      }
bindToDataLoaderOkWith10sDelay(int32_t mountId,const DataLoaderParamsParcel & params,int bindDelayMs,const sp<IDataLoaderStatusListener> & listener,bool * _aidl_return)300      binder::Status bindToDataLoaderOkWith10sDelay(int32_t mountId,
301                                                    const DataLoaderParamsParcel& params,
302                                                    int bindDelayMs,
303                                                    const sp<IDataLoaderStatusListener>& listener,
304                                                    bool* _aidl_return) {
305          CHECK(100 * 9 * 9 <= bindDelayMs && bindDelayMs <= 100 * 11 * 11) << bindDelayMs;
306          return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
307      }
bindToDataLoaderOkWith100sDelay(int32_t mountId,const DataLoaderParamsParcel & params,int bindDelayMs,const sp<IDataLoaderStatusListener> & listener,bool * _aidl_return)308      binder::Status bindToDataLoaderOkWith100sDelay(int32_t mountId,
309                                                     const DataLoaderParamsParcel& params,
310                                                     int bindDelayMs,
311                                                     const sp<IDataLoaderStatusListener>& listener,
312                                                     bool* _aidl_return) {
313          CHECK(100 * 9 * 9 * 9 < bindDelayMs && bindDelayMs < 100 * 11 * 11 * 11) << bindDelayMs;
314          return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
315      }
bindToDataLoaderOkWith1000sDelay(int32_t mountId,const DataLoaderParamsParcel & params,int bindDelayMs,const sp<IDataLoaderStatusListener> & listener,bool * _aidl_return)316      binder::Status bindToDataLoaderOkWith1000sDelay(int32_t mountId,
317                                                      const DataLoaderParamsParcel& params,
318                                                      int bindDelayMs,
319                                                      const sp<IDataLoaderStatusListener>& listener,
320                                                      bool* _aidl_return) {
321          CHECK(100 * 9 * 9 * 9 * 9 < bindDelayMs && bindDelayMs < 100 * 11 * 11 * 11 * 11)
322                  << bindDelayMs;
323          return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
324      }
bindToDataLoaderOkWith10000sDelay(int32_t mountId,const DataLoaderParamsParcel & params,int bindDelayMs,const sp<IDataLoaderStatusListener> & listener,bool * _aidl_return)325      binder::Status bindToDataLoaderOkWith10000sDelay(int32_t mountId,
326                                                       const DataLoaderParamsParcel& params,
327                                                       int bindDelayMs,
328                                                       const sp<IDataLoaderStatusListener>& listener,
329                                                       bool* _aidl_return) {
330          CHECK(100 * 9 * 9 * 9 * 9 * 9 < bindDelayMs && bindDelayMs < 100 * 11 * 11 * 11 * 11 * 11)
331                  << bindDelayMs;
332          return bindToDataLoaderOk(mountId, params, bindDelayMs, listener, _aidl_return);
333      }
334  
getDataLoaderOk(int32_t mountId,sp<IDataLoader> * _aidl_return)335      binder::Status getDataLoaderOk(int32_t mountId, sp<IDataLoader>* _aidl_return) {
336          *_aidl_return = mDataLoader;
337          return binder::Status::ok();
338      }
setDataLoaderStatusCreated()339      void setDataLoaderStatusCreated() {
340          mListener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_CREATED);
341      }
setDataLoaderStatusStarted()342      void setDataLoaderStatusStarted() {
343          mListener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_STARTED);
344      }
setDataLoaderStatusDestroyed()345      void setDataLoaderStatusDestroyed() {
346          mListener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_DESTROYED);
347      }
setDataLoaderStatusUnavailable()348      void setDataLoaderStatusUnavailable() {
349          mListener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_UNAVAILABLE);
350      }
setDataLoaderStatusUnrecoverable()351      void setDataLoaderStatusUnrecoverable() {
352          mListener->onStatusChanged(mId, IDataLoaderStatusListener::DATA_LOADER_UNRECOVERABLE);
353      }
unbindFromDataLoaderOk(int32_t id)354      binder::Status unbindFromDataLoaderOk(int32_t id) {
355          mBindDelayMs = -1;
356          if (mDataLoader) {
357              if (auto status = mDataLoader->destroy(id); !status.isOk()) {
358                  return status;
359              }
360              mDataLoader = nullptr;
361          } else if (mListener) {
362              mListener->onStatusChanged(id, IDataLoaderStatusListener::DATA_LOADER_DESTROYED);
363          }
364          return binder::Status::ok();
365      }
366  
bindDelayMs() const367      int bindDelayMs() const { return mBindDelayMs; }
368  
369  private:
370      int mId = -1;
371      int mBindDelayMs = -1;
372      sp<IDataLoaderStatusListener> mListener;
373      sp<IDataLoader> mDataLoader;
374      sp<IDataLoader> mDataLoaderHolder;
375  };
376  
377  class MockIncFs : public IncFsWrapper {
378  public:
379      MOCK_CONST_METHOD0(features, Features());
380      MOCK_CONST_METHOD1(listExistingMounts, void(const ExistingMountCallback& cb));
381      MOCK_CONST_METHOD1(openMount, Control(std::string_view path));
382      MOCK_CONST_METHOD4(createControl,
383                         Control(IncFsFd cmd, IncFsFd pendingReads, IncFsFd logs,
384                                 IncFsFd blocksWritten));
385      MOCK_CONST_METHOD5(makeFile,
386                         ErrorCode(const Control& control, std::string_view path, int mode, FileId id,
387                                   NewFileParams params));
388      MOCK_CONST_METHOD4(makeMappedFile,
389                         ErrorCode(const Control& control, std::string_view path, int mode,
390                                   NewMappedFileParams params));
391      MOCK_CONST_METHOD3(makeDir, ErrorCode(const Control& control, std::string_view path, int mode));
392      MOCK_CONST_METHOD3(makeDirs,
393                         ErrorCode(const Control& control, std::string_view path, int mode));
394      MOCK_CONST_METHOD2(getMetadata, RawMetadata(const Control& control, FileId fileid));
395      MOCK_CONST_METHOD2(getMetadata, RawMetadata(const Control& control, std::string_view path));
396      MOCK_CONST_METHOD2(getFileId, FileId(const Control& control, std::string_view path));
397      MOCK_CONST_METHOD2(countFilledBlocks,
398                         std::pair<IncFsBlockIndex, IncFsBlockIndex>(const Control& control,
399                                                                     std::string_view path));
400      MOCK_CONST_METHOD2(isFileFullyLoaded,
401                         incfs::LoadingState(const Control& control, std::string_view path));
402      MOCK_CONST_METHOD2(isFileFullyLoaded, incfs::LoadingState(const Control& control, FileId id));
403      MOCK_CONST_METHOD1(isEverythingFullyLoaded, incfs::LoadingState(const Control& control));
404      MOCK_CONST_METHOD3(link,
405                         ErrorCode(const Control& control, std::string_view from,
406                                   std::string_view to));
407      MOCK_CONST_METHOD2(unlink, ErrorCode(const Control& control, std::string_view path));
408      MOCK_CONST_METHOD2(openForSpecialOps, UniqueFd(const Control& control, FileId id));
409      MOCK_CONST_METHOD1(writeBlocks, ErrorCode(std::span<const DataBlock> blocks));
410      MOCK_CONST_METHOD3(reserveSpace, ErrorCode(const Control& control, FileId id, IncFsSize size));
411      MOCK_CONST_METHOD3(waitForPendingReads,
412                         WaitResult(const Control& control, std::chrono::milliseconds timeout,
413                                    std::vector<incfs::ReadInfoWithUid>* pendingReadsBuffer));
414      MOCK_CONST_METHOD2(setUidReadTimeouts,
415                         ErrorCode(const Control& control,
416                                   const std::vector<PerUidReadTimeouts>& perUidReadTimeouts));
417      MOCK_CONST_METHOD2(forEachFile, ErrorCode(const Control& control, FileCallback cb));
418      MOCK_CONST_METHOD2(forEachIncompleteFile, ErrorCode(const Control& control, FileCallback cb));
419      MOCK_CONST_METHOD1(getMetrics, std::optional<Metrics>(std::string_view path));
420      MOCK_CONST_METHOD1(getLastReadError, std::optional<LastReadError>(const Control& control));
421  
MockIncFs()422      MockIncFs() {
423          ON_CALL(*this, listExistingMounts(_)).WillByDefault(Return());
424          ON_CALL(*this, reserveSpace(_, _, _)).WillByDefault(Return(0));
425      }
426  
makeFileFails()427      void makeFileFails() { ON_CALL(*this, makeFile(_, _, _, _, _)).WillByDefault(Return(-1)); }
makeFileSuccess()428      void makeFileSuccess() { ON_CALL(*this, makeFile(_, _, _, _, _)).WillByDefault(Return(0)); }
429  
countFilledBlocksSuccess()430      void countFilledBlocksSuccess() {
431          ON_CALL(*this, countFilledBlocks(_, _)).WillByDefault(Return(std::make_pair(1, 2)));
432      }
433  
countFilledBlocksFullyLoaded()434      void countFilledBlocksFullyLoaded() {
435          ON_CALL(*this, countFilledBlocks(_, _)).WillByDefault(Return(std::make_pair(10000, 10000)));
436      }
437  
countFilledBlocksFails()438      void countFilledBlocksFails() {
439          ON_CALL(*this, countFilledBlocks(_, _)).WillByDefault(Return(std::make_pair(-1, -1)));
440      }
441  
countFilledBlocksEmpty()442      void countFilledBlocksEmpty() {
443          ON_CALL(*this, countFilledBlocks(_, _)).WillByDefault(Return(std::make_pair(0, 0)));
444      }
445  
openMountSuccess()446      void openMountSuccess() {
447          ON_CALL(*this, openMount(_)).WillByDefault(Invoke(this, &MockIncFs::openMountForHealth));
448      }
449  
450      // 1000ms
waitForPendingReadsSuccess(uint64_t ts=0)451      void waitForPendingReadsSuccess(uint64_t ts = 0) {
452          ON_CALL(*this, waitForPendingReads(_, _, _))
453                  .WillByDefault(
454                          Invoke([ts](const Control& control, std::chrono::milliseconds timeout,
455                                      std::vector<incfs::ReadInfoWithUid>* pendingReadsBuffer) {
456                              pendingReadsBuffer->push_back({.bootClockTsUs = ts});
457                              return android::incfs::WaitResult::HaveData;
458                          }));
459      }
460  
waitForPendingReadsTimeout()461      void waitForPendingReadsTimeout() {
462          ON_CALL(*this, waitForPendingReads(_, _, _))
463                  .WillByDefault(Return(android::incfs::WaitResult::Timeout));
464      }
465  
466      static constexpr auto kPendingReadsFd = 42;
openMountForHealth(std::string_view)467      Control openMountForHealth(std::string_view) {
468          return UniqueControl(IncFs_CreateControl(-1, kPendingReadsFd, -1, -1));
469      }
470  
getMountInfoMetadata(const Control & control,std::string_view path)471      RawMetadata getMountInfoMetadata(const Control& control, std::string_view path) {
472          metadata::Mount m;
473          m.mutable_storage()->set_id(100);
474          m.mutable_loader()->set_package_name("com.test");
475          m.mutable_loader()->set_arguments("com.uri");
476          const auto metadata = m.SerializeAsString();
477          static_cast<void>(m.mutable_loader()->release_arguments());
478          static_cast<void>(m.mutable_loader()->release_package_name());
479          return {metadata.begin(), metadata.end()};
480      }
getStorageMetadata(const Control & control,std::string_view path)481      RawMetadata getStorageMetadata(const Control& control, std::string_view path) {
482          metadata::Storage st;
483          st.set_id(100);
484          auto metadata = st.SerializeAsString();
485          return {metadata.begin(), metadata.end()};
486      }
getBindPointMetadata(const Control & control,std::string_view path)487      RawMetadata getBindPointMetadata(const Control& control, std::string_view path) {
488          metadata::BindPoint bp;
489          std::string destPath = "dest";
490          std::string srcPath = "src";
491          bp.set_storage_id(100);
492          bp.set_allocated_dest_path(&destPath);
493          bp.set_allocated_source_subdir(&srcPath);
494          const auto metadata = bp.SerializeAsString();
495          static_cast<void>(bp.release_source_subdir());
496          static_cast<void>(bp.release_dest_path());
497          return std::vector<char>(metadata.begin(), metadata.end());
498      }
499  };
500  
501  class MockAppOpsManager : public AppOpsManagerWrapper {
502  public:
503      MOCK_CONST_METHOD3(checkPermission, binder::Status(const char*, const char*, const char*));
504      MOCK_METHOD3(startWatchingMode, void(int32_t, const String16&, const sp<IAppOpsCallback>&));
505      MOCK_METHOD1(stopWatchingMode, void(const sp<IAppOpsCallback>&));
506  
checkPermissionSuccess()507      void checkPermissionSuccess() {
508          ON_CALL(*this, checkPermission(_, _, _)).WillByDefault(Return(android::incremental::Ok()));
509      }
checkPermissionNoCrossUsers()510      void checkPermissionNoCrossUsers() {
511          ON_CALL(*this,
512                  checkPermission("android.permission.LOADER_USAGE_STATS",
513                                  "android:loader_usage_stats", _))
514                  .WillByDefault(Return(android::incremental::Ok()));
515          ON_CALL(*this, checkPermission("android.permission.INTERACT_ACROSS_USERS", nullptr, _))
516                  .WillByDefault(
517                          Return(android::incremental::Exception(binder::Status::EX_SECURITY, {})));
518      }
checkPermissionFails()519      void checkPermissionFails() {
520          ON_CALL(*this, checkPermission(_, _, _))
521                  .WillByDefault(
522                          Return(android::incremental::Exception(binder::Status::EX_SECURITY, {})));
523      }
initializeStartWatchingMode()524      void initializeStartWatchingMode() {
525          ON_CALL(*this, startWatchingMode(_, _, _))
526                  .WillByDefault(Invoke(this, &MockAppOpsManager::storeCallback));
527      }
storeCallback(int32_t,const String16 &,const sp<IAppOpsCallback> & cb)528      void storeCallback(int32_t, const String16&, const sp<IAppOpsCallback>& cb) {
529          mStoredCallback = cb;
530      }
531  
532      sp<IAppOpsCallback> mStoredCallback;
533  };
534  
535  class MockJniWrapper : public JniWrapper {
536  public:
537      MOCK_CONST_METHOD0(initializeForCurrentThread, void());
538  
MockJniWrapper()539      MockJniWrapper() { EXPECT_CALL(*this, initializeForCurrentThread()).Times(2); }
540  };
541  
542  class MockLooperWrapper : public LooperWrapper {
543  public:
544      MOCK_METHOD5(addFd, int(int, int, int, android::Looper_callbackFunc, void*));
545      MOCK_METHOD1(removeFd, int(int));
546      MOCK_METHOD0(wake, void());
547      MOCK_METHOD1(pollAll, int(int));
548  
MockLooperWrapper()549      MockLooperWrapper() {
550          ON_CALL(*this, addFd(_, _, _, _, _))
551                  .WillByDefault(Invoke(this, &MockLooperWrapper::storeCallback));
552          ON_CALL(*this, removeFd(_)).WillByDefault(Invoke(this, &MockLooperWrapper::clearCallback));
553          ON_CALL(*this, pollAll(_)).WillByDefault(Invoke(this, &MockLooperWrapper::wait10Ms));
554      }
555  
storeCallback(int,int,int,android::Looper_callbackFunc callback,void * data)556      int storeCallback(int, int, int, android::Looper_callbackFunc callback, void* data) {
557          mCallback = callback;
558          mCallbackData = data;
559          return 0;
560      }
561  
clearCallback(int)562      int clearCallback(int) {
563          mCallback = nullptr;
564          mCallbackData = nullptr;
565          return 0;
566      }
567  
wait10Ms(int)568      int wait10Ms(int) {
569          // This is called from a loop in runCmdLooper.
570          // Sleeping for 10ms only to avoid busy looping.
571          std::this_thread::sleep_for(10ms);
572          return 0;
573      }
574  
575      android::Looper_callbackFunc mCallback = nullptr;
576      void* mCallbackData = nullptr;
577  };
578  
579  class MockTimedQueueWrapper : public TimedQueueWrapper {
580  public:
581      MOCK_METHOD3(addJob, void(MountId, Milliseconds, Job));
582      MOCK_METHOD1(removeJobs, void(MountId));
583      MOCK_METHOD0(stop, void());
584  
MockTimedQueueWrapper()585      MockTimedQueueWrapper() {
586          ON_CALL(*this, addJob(_, _, _))
587                  .WillByDefault(Invoke(this, &MockTimedQueueWrapper::storeJob));
588          ON_CALL(*this, removeJobs(_)).WillByDefault(Invoke(this, &MockTimedQueueWrapper::clearJob));
589      }
590  
storeJob(MountId id,Milliseconds after,Job what)591      void storeJob(MountId id, Milliseconds after, Job what) {
592          mId = id;
593          mAfter = after;
594          mWhat = std::move(what);
595      }
596  
clearJob(MountId id)597      void clearJob(MountId id) {
598          if (mId == id) {
599              mAfter = {};
600              mWhat = {};
601          }
602      }
603  
604      MountId mId = -1;
605      Milliseconds mAfter;
606      Job mWhat;
607  };
608  
609  class MockFsWrapper : public FsWrapper {
610  public:
611      MOCK_CONST_METHOD2(listFilesRecursive, void(std::string_view, FileCallback));
hasNoFile()612      void hasNoFile() { ON_CALL(*this, listFilesRecursive(_, _)).WillByDefault(Return()); }
hasFiles()613      void hasFiles() {
614          ON_CALL(*this, listFilesRecursive(_, _))
615                  .WillByDefault(Invoke(this, &MockFsWrapper::fakeFiles));
616      }
fakeFiles(std::string_view directoryPath,FileCallback onFile)617      void fakeFiles(std::string_view directoryPath, FileCallback onFile) {
618          for (auto file : {"base.apk", "split.apk", "lib/a.so"}) {
619              if (!onFile(file)) break;
620          }
621      }
622  };
623  
624  class MockClockWrapper : public ClockWrapper {
625  public:
626      MOCK_CONST_METHOD0(now, TimePoint());
627  
start()628      void start() { ON_CALL(*this, now()).WillByDefault(Invoke(this, &MockClockWrapper::getClock)); }
629  
630      template <class Delta>
advance(Delta delta)631      void advance(Delta delta) {
632          mClock += delta;
633      }
634  
advanceMs(int deltaMs)635      void advanceMs(int deltaMs) { mClock += std::chrono::milliseconds(deltaMs); }
636  
getClock() const637      TimePoint getClock() const { return mClock; }
getClockMono() const638      std::optional<timespec> getClockMono() const {
639          const auto nsSinceEpoch =
640                  std::chrono::duration_cast<std::chrono::nanoseconds>(mClock.time_since_epoch())
641                          .count();
642          timespec ts = {.tv_sec = static_cast<time_t>(nsSinceEpoch / 1000000000LL),
643                         .tv_nsec = static_cast<long>(nsSinceEpoch % 1000000000LL)};
644          return ts;
645      }
646  
647      TimePoint mClock = Clock::now();
648  };
649  
650  class MockStorageHealthListener : public os::incremental::BnStorageHealthListener {
651  public:
652      MOCK_METHOD2(onHealthStatus, binder::Status(int32_t storageId, int32_t status));
653  
MockStorageHealthListener()654      MockStorageHealthListener() {
655          ON_CALL(*this, onHealthStatus(_, _))
656                  .WillByDefault(Invoke(this, &MockStorageHealthListener::storeStorageIdAndStatus));
657      }
658  
storeStorageIdAndStatus(int32_t storageId,int32_t status)659      binder::Status storeStorageIdAndStatus(int32_t storageId, int32_t status) {
660          mStorageId = storageId;
661          mStatus = status;
662          return binder::Status::ok();
663      }
664  
665      int32_t mStorageId = -1;
666      int32_t mStatus = -1;
667  };
668  
669  class MockStorageLoadingProgressListener : public IStorageLoadingProgressListener {
670  public:
671      MockStorageLoadingProgressListener() = default;
672      MOCK_METHOD2(onStorageLoadingProgressChanged,
673                   binder::Status(int32_t storageId, float progress));
674      MOCK_METHOD0(onAsBinder, IBinder*());
675  };
676  
677  class MockServiceManager : public ServiceManagerWrapper {
678  public:
MockServiceManager(std::unique_ptr<MockVoldService> vold,std::unique_ptr<MockDataLoaderManager> dataLoaderManager,std::unique_ptr<MockIncFs> incfs,std::unique_ptr<MockAppOpsManager> appOpsManager,std::unique_ptr<MockJniWrapper> jni,std::unique_ptr<MockLooperWrapper> looper,std::unique_ptr<MockTimedQueueWrapper> timedQueue,std::unique_ptr<MockTimedQueueWrapper> progressUpdateJobQueue,std::unique_ptr<MockFsWrapper> fs,std::unique_ptr<MockClockWrapper> clock)679      MockServiceManager(std::unique_ptr<MockVoldService> vold,
680                         std::unique_ptr<MockDataLoaderManager> dataLoaderManager,
681                         std::unique_ptr<MockIncFs> incfs,
682                         std::unique_ptr<MockAppOpsManager> appOpsManager,
683                         std::unique_ptr<MockJniWrapper> jni,
684                         std::unique_ptr<MockLooperWrapper> looper,
685                         std::unique_ptr<MockTimedQueueWrapper> timedQueue,
686                         std::unique_ptr<MockTimedQueueWrapper> progressUpdateJobQueue,
687                         std::unique_ptr<MockFsWrapper> fs, std::unique_ptr<MockClockWrapper> clock)
688            : mVold(std::move(vold)),
689              mDataLoaderManager(std::move(dataLoaderManager)),
690              mIncFs(std::move(incfs)),
691              mAppOpsManager(std::move(appOpsManager)),
692              mJni(std::move(jni)),
693              mLooper(std::move(looper)),
694              mTimedQueue(std::move(timedQueue)),
695              mProgressUpdateJobQueue(std::move(progressUpdateJobQueue)),
696              mFs(std::move(fs)),
697              mClock(std::move(clock)) {}
getVoldService()698      std::unique_ptr<VoldServiceWrapper> getVoldService() final { return std::move(mVold); }
getDataLoaderManager()699      std::unique_ptr<DataLoaderManagerWrapper> getDataLoaderManager() final {
700          return std::move(mDataLoaderManager);
701      }
getIncFs()702      std::unique_ptr<IncFsWrapper> getIncFs() final { return std::move(mIncFs); }
getAppOpsManager()703      std::unique_ptr<AppOpsManagerWrapper> getAppOpsManager() final {
704          return std::move(mAppOpsManager);
705      }
getJni()706      std::unique_ptr<JniWrapper> getJni() final { return std::move(mJni); }
getLooper()707      std::unique_ptr<LooperWrapper> getLooper() final { return std::move(mLooper); }
getTimedQueue()708      std::unique_ptr<TimedQueueWrapper> getTimedQueue() final { return std::move(mTimedQueue); }
getProgressUpdateJobQueue()709      std::unique_ptr<TimedQueueWrapper> getProgressUpdateJobQueue() final {
710          return std::move(mProgressUpdateJobQueue);
711      }
getFs()712      std::unique_ptr<FsWrapper> getFs() final { return std::move(mFs); }
getClock()713      std::unique_ptr<ClockWrapper> getClock() final { return std::move(mClock); }
714  
715  private:
716      std::unique_ptr<MockVoldService> mVold;
717      std::unique_ptr<MockDataLoaderManager> mDataLoaderManager;
718      std::unique_ptr<MockIncFs> mIncFs;
719      std::unique_ptr<MockAppOpsManager> mAppOpsManager;
720      std::unique_ptr<MockJniWrapper> mJni;
721      std::unique_ptr<MockLooperWrapper> mLooper;
722      std::unique_ptr<MockTimedQueueWrapper> mTimedQueue;
723      std::unique_ptr<MockTimedQueueWrapper> mProgressUpdateJobQueue;
724      std::unique_ptr<MockFsWrapper> mFs;
725      std::unique_ptr<MockClockWrapper> mClock;
726  };
727  
728  // --- IncrementalServiceTest ---
729  
730  class IncrementalServiceTest : public testing::Test {
731  public:
SetUp()732      void SetUp() override {
733          auto vold = std::make_unique<NiceMock<MockVoldService>>();
734          mVold = vold.get();
735          sp<NiceMock<MockDataLoader>> dataLoader{new NiceMock<MockDataLoader>};
736          mDataLoader = dataLoader.get();
737          auto dataloaderManager = std::make_unique<NiceMock<MockDataLoaderManager>>(dataLoader);
738          mDataLoaderManager = dataloaderManager.get();
739          auto incFs = std::make_unique<NiceMock<MockIncFs>>();
740          mIncFs = incFs.get();
741          auto appOps = std::make_unique<NiceMock<MockAppOpsManager>>();
742          mAppOpsManager = appOps.get();
743          auto jni = std::make_unique<NiceMock<MockJniWrapper>>();
744          mJni = jni.get();
745          auto looper = std::make_unique<NiceMock<MockLooperWrapper>>();
746          mLooper = looper.get();
747          auto timedQueue = std::make_unique<NiceMock<MockTimedQueueWrapper>>();
748          mTimedQueue = timedQueue.get();
749          auto progressUpdateJobQueue = std::make_unique<NiceMock<MockTimedQueueWrapper>>();
750          mProgressUpdateJobQueue = progressUpdateJobQueue.get();
751          auto fs = std::make_unique<NiceMock<MockFsWrapper>>();
752          mFs = fs.get();
753          auto clock = std::make_unique<NiceMock<MockClockWrapper>>();
754          mClock = clock.get();
755          mIncrementalService = std::make_unique<
756                  IncrementalService>(MockServiceManager(std::move(vold),
757                                                         std::move(dataloaderManager),
758                                                         std::move(incFs), std::move(appOps),
759                                                         std::move(jni), std::move(looper),
760                                                         std::move(timedQueue),
761                                                         std::move(progressUpdateJobQueue),
762                                                         std::move(fs), std::move(clock)),
763                                      mRootDir.path);
764          mDataLoaderParcel.packageName = "com.test";
765          mDataLoaderParcel.arguments = "uri";
766          mDataLoaderManager->unbindFromDataLoaderSuccess();
767          mIncrementalService->onSystemReady();
768          mClock->start();
769          setupSuccess();
770      }
771  
setUpExistingMountDir(const std::string & rootDir)772      void setUpExistingMountDir(const std::string& rootDir) {
773          const auto dir = rootDir + "/dir1";
774          const auto mountDir = dir + "/mount";
775          const auto backingDir = dir + "/backing_store";
776          const auto storageDir = mountDir + "/st0";
777          ASSERT_EQ(0, mkdir(dir.c_str(), 0755));
778          ASSERT_EQ(0, mkdir(mountDir.c_str(), 0755));
779          ASSERT_EQ(0, mkdir(backingDir.c_str(), 0755));
780          ASSERT_EQ(0, mkdir(storageDir.c_str(), 0755));
781          const auto mountInfoFile = rootDir + "/dir1/mount/.info";
782          const auto mountPointsFile = rootDir + "/dir1/mount/.mountpoint.abcd";
783          ASSERT_TRUE(base::WriteStringToFile("info", mountInfoFile));
784          ASSERT_TRUE(base::WriteStringToFile("mounts", mountPointsFile));
785          ON_CALL(*mIncFs, getMetadata(_, std::string_view(mountInfoFile)))
786                  .WillByDefault(Invoke(mIncFs, &MockIncFs::getMountInfoMetadata));
787          ON_CALL(*mIncFs, getMetadata(_, std::string_view(mountPointsFile)))
788                  .WillByDefault(Invoke(mIncFs, &MockIncFs::getBindPointMetadata));
789          ON_CALL(*mIncFs, getMetadata(_, std::string_view(rootDir + "/dir1/mount/st0")))
790                  .WillByDefault(Invoke(mIncFs, &MockIncFs::getStorageMetadata));
791      }
792  
setupSuccess()793      void setupSuccess() {
794          mVold->mountIncFsSuccess();
795          mIncFs->makeFileSuccess();
796          mVold->bindMountSuccess();
797          mDataLoaderManager->bindToDataLoaderSuccess();
798          mDataLoaderManager->getDataLoaderSuccess();
799      }
800  
checkHealthMetrics(int storageId,long expectedMillisSinceOldestPendingRead,int expectedStorageHealthStatusCode)801      void checkHealthMetrics(int storageId, long expectedMillisSinceOldestPendingRead,
802                              int expectedStorageHealthStatusCode) {
803          android::os::PersistableBundle result{};
804          mIncrementalService->getMetrics(storageId, &result);
805          ASSERT_EQ(6, (int)result.size());
806          int64_t millisSinceOldestPendingRead = -1;
807          ASSERT_TRUE(result.getLong(String16(BnIncrementalService::
808                                                      METRICS_MILLIS_SINCE_OLDEST_PENDING_READ()
809                                                              .c_str()),
810                                     &millisSinceOldestPendingRead));
811          // Allow 10ms.
812          ASSERT_LE(expectedMillisSinceOldestPendingRead, millisSinceOldestPendingRead);
813          ASSERT_GE(expectedMillisSinceOldestPendingRead + 10, millisSinceOldestPendingRead);
814          int storageHealthStatusCode = -1;
815          ASSERT_TRUE(
816                  result.getInt(String16(BnIncrementalService::METRICS_STORAGE_HEALTH_STATUS_CODE()
817                                                 .c_str()),
818                                &storageHealthStatusCode));
819          ASSERT_EQ(expectedStorageHealthStatusCode, storageHealthStatusCode);
820          int dataLoaderStatusCode = -1;
821          ASSERT_TRUE(result.getInt(String16(BnIncrementalService::METRICS_DATA_LOADER_STATUS_CODE()
822                                                     .c_str()),
823                                    &dataLoaderStatusCode));
824          ASSERT_EQ(IDataLoaderStatusListener::DATA_LOADER_STARTED, dataLoaderStatusCode);
825      }
826  
checkBindingMetrics(int storageId,int64_t expectedMillisSinceLastDataLoaderBind,int64_t expectedDataLoaderBindDelayMillis)827      void checkBindingMetrics(int storageId, int64_t expectedMillisSinceLastDataLoaderBind,
828                               int64_t expectedDataLoaderBindDelayMillis) {
829          android::os::PersistableBundle result{};
830          mIncrementalService->getMetrics(storageId, &result);
831          ASSERT_EQ(6, (int)result.size());
832          int dataLoaderStatus = -1;
833          ASSERT_TRUE(result.getInt(String16(BnIncrementalService::METRICS_DATA_LOADER_STATUS_CODE()
834                                                     .c_str()),
835                                    &dataLoaderStatus));
836          ASSERT_EQ(IDataLoaderStatusListener::DATA_LOADER_STARTED, dataLoaderStatus);
837          int64_t millisSinceLastDataLoaderBind = -1;
838          ASSERT_TRUE(result.getLong(String16(BnIncrementalService::
839                                                      METRICS_MILLIS_SINCE_LAST_DATA_LOADER_BIND()
840                                                              .c_str()),
841                                     &millisSinceLastDataLoaderBind));
842          ASSERT_EQ(expectedMillisSinceLastDataLoaderBind, millisSinceLastDataLoaderBind);
843          int64_t dataLoaderBindDelayMillis = -1;
844          ASSERT_TRUE(
845                  result.getLong(String16(
846                                         BnIncrementalService::METRICS_DATA_LOADER_BIND_DELAY_MILLIS()
847                                                 .c_str()),
848                                 &dataLoaderBindDelayMillis));
849          ASSERT_EQ(expectedDataLoaderBindDelayMillis, dataLoaderBindDelayMillis);
850      }
851  
852  protected:
853      NiceMock<MockVoldService>* mVold = nullptr;
854      NiceMock<MockIncFs>* mIncFs = nullptr;
855      NiceMock<MockDataLoaderManager>* mDataLoaderManager = nullptr;
856      NiceMock<MockAppOpsManager>* mAppOpsManager = nullptr;
857      NiceMock<MockJniWrapper>* mJni = nullptr;
858      NiceMock<MockLooperWrapper>* mLooper = nullptr;
859      NiceMock<MockTimedQueueWrapper>* mTimedQueue = nullptr;
860      NiceMock<MockTimedQueueWrapper>* mProgressUpdateJobQueue = nullptr;
861      NiceMock<MockFsWrapper>* mFs = nullptr;
862      NiceMock<MockClockWrapper>* mClock = nullptr;
863      NiceMock<MockDataLoader>* mDataLoader = nullptr;
864      std::unique_ptr<IncrementalService> mIncrementalService;
865      TemporaryDir mRootDir;
866      DataLoaderParamsParcel mDataLoaderParcel;
867  };
868  
TEST_F(IncrementalServiceTest,testCreateStorageMountIncFsFails)869  TEST_F(IncrementalServiceTest, testCreateStorageMountIncFsFails) {
870      mVold->mountIncFsFails();
871      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
872      TemporaryDir tempDir;
873      int storageId =
874              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
875                                                 IncrementalService::CreateOptions::CreateNew);
876      ASSERT_LT(storageId, 0);
877  }
878  
TEST_F(IncrementalServiceTest,testCreateStorageMountIncFsInvalidControlParcel)879  TEST_F(IncrementalServiceTest, testCreateStorageMountIncFsInvalidControlParcel) {
880      mVold->mountIncFsInvalidControlParcel();
881      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
882      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
883      TemporaryDir tempDir;
884      int storageId =
885              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
886                                                 IncrementalService::CreateOptions::CreateNew);
887      ASSERT_LT(storageId, 0);
888  }
889  
TEST_F(IncrementalServiceTest,testCreateStorageMakeFileFails)890  TEST_F(IncrementalServiceTest, testCreateStorageMakeFileFails) {
891      mVold->mountIncFsSuccess();
892      mIncFs->makeFileFails();
893      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
894      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
895      EXPECT_CALL(*mVold, unmountIncFs(_));
896      TemporaryDir tempDir;
897      int storageId =
898              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
899                                                 IncrementalService::CreateOptions::CreateNew);
900      ASSERT_LT(storageId, 0);
901  }
902  
TEST_F(IncrementalServiceTest,testCreateStorageBindMountFails)903  TEST_F(IncrementalServiceTest, testCreateStorageBindMountFails) {
904      mVold->mountIncFsSuccess();
905      mIncFs->makeFileSuccess();
906      mVold->bindMountFails();
907      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(0);
908      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
909      EXPECT_CALL(*mVold, unmountIncFs(_));
910      TemporaryDir tempDir;
911      int storageId =
912              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
913                                                 IncrementalService::CreateOptions::CreateNew);
914      ASSERT_LT(storageId, 0);
915  }
916  
TEST_F(IncrementalServiceTest,testCreateStoragePrepareDataLoaderFails)917  TEST_F(IncrementalServiceTest, testCreateStoragePrepareDataLoaderFails) {
918      mVold->mountIncFsSuccess();
919      mIncFs->makeFileSuccess();
920      mVold->bindMountSuccess();
921      mDataLoaderManager->bindToDataLoaderFails();
922      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
923      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
924      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(0);
925      EXPECT_CALL(*mDataLoader, start(_)).Times(0);
926      EXPECT_CALL(*mDataLoader, destroy(_)).Times(0);
927      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
928      TemporaryDir tempDir;
929      int storageId =
930              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
931                                                 IncrementalService::CreateOptions::CreateNew);
932      ASSERT_GE(storageId, 0);
933      mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {}, {}, {});
934  }
935  
TEST_F(IncrementalServiceTest,testDeleteStorageSuccess)936  TEST_F(IncrementalServiceTest, testDeleteStorageSuccess) {
937      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
938      EXPECT_CALL(*mDataLoader, start(_)).Times(1);
939      EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
940      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
941      TemporaryDir tempDir;
942      int storageId =
943              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
944                                                 IncrementalService::CreateOptions::CreateNew);
945      ASSERT_GE(storageId, 0);
946      mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {}, {}, {});
947      mIncrementalService->deleteStorage(storageId);
948  }
949  
TEST_F(IncrementalServiceTest,testDataLoaderDestroyedAndDelayed)950  TEST_F(IncrementalServiceTest, testDataLoaderDestroyedAndDelayed) {
951      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(7);
952      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
953      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(7);
954      EXPECT_CALL(*mDataLoader, start(_)).Times(7);
955      EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
956      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
957      TemporaryDir tempDir;
958      int storageId =
959              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
960                                                 IncrementalService::CreateOptions::CreateNew);
961      ASSERT_GE(storageId, 0);
962      mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {}, {}, {});
963  
964      // Simulated crash/other connection breakage.
965  
966      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
967              .WillByDefault(Invoke(mDataLoaderManager,
968                                    &MockDataLoaderManager::bindToDataLoaderOkWith1sDelay));
969      checkBindingMetrics(storageId, 0, 0);
970      mClock->advanceMs(mDataLoaderManager->bindDelayMs());
971      checkBindingMetrics(storageId, 0, 0);
972      mDataLoaderManager->setDataLoaderStatusDestroyed();
973  
974      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
975              .WillByDefault(Invoke(mDataLoaderManager,
976                                    &MockDataLoaderManager::bindToDataLoaderOkWith10sDelay));
977      checkBindingMetrics(storageId, 0, mDataLoaderManager->bindDelayMs());
978      mClock->advanceMs(mDataLoaderManager->bindDelayMs());
979      checkBindingMetrics(storageId, mDataLoaderManager->bindDelayMs(),
980                          mDataLoaderManager->bindDelayMs());
981      mDataLoaderManager->setDataLoaderStatusDestroyed();
982  
983      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
984              .WillByDefault(Invoke(mDataLoaderManager,
985                                    &MockDataLoaderManager::bindToDataLoaderOkWith100sDelay));
986      checkBindingMetrics(storageId, 0, mDataLoaderManager->bindDelayMs());
987      mClock->advanceMs(mDataLoaderManager->bindDelayMs());
988      checkBindingMetrics(storageId, mDataLoaderManager->bindDelayMs(),
989                          mDataLoaderManager->bindDelayMs());
990      mDataLoaderManager->setDataLoaderStatusDestroyed();
991  
992      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
993              .WillByDefault(Invoke(mDataLoaderManager,
994                                    &MockDataLoaderManager::bindToDataLoaderOkWith1000sDelay));
995      checkBindingMetrics(storageId, 0, mDataLoaderManager->bindDelayMs());
996      mClock->advanceMs(mDataLoaderManager->bindDelayMs());
997      checkBindingMetrics(storageId, mDataLoaderManager->bindDelayMs(),
998                          mDataLoaderManager->bindDelayMs());
999      mDataLoaderManager->setDataLoaderStatusDestroyed();
1000  
1001      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1002              .WillByDefault(Invoke(mDataLoaderManager,
1003                                    &MockDataLoaderManager::bindToDataLoaderOkWith10000sDelay));
1004      checkBindingMetrics(storageId, 0, mDataLoaderManager->bindDelayMs());
1005      // Try the reduced delay, just in case.
1006      mClock->advanceMs(mDataLoaderManager->bindDelayMs() / 2);
1007      checkBindingMetrics(storageId, mDataLoaderManager->bindDelayMs() / 2,
1008                          mDataLoaderManager->bindDelayMs());
1009      mDataLoaderManager->setDataLoaderStatusDestroyed();
1010  
1011      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1012              .WillByDefault(Invoke(mDataLoaderManager,
1013                                    &MockDataLoaderManager::bindToDataLoaderOkWith10000sDelay));
1014      checkBindingMetrics(storageId, 0, mDataLoaderManager->bindDelayMs());
1015      mClock->advanceMs(mDataLoaderManager->bindDelayMs());
1016      checkBindingMetrics(storageId, mDataLoaderManager->bindDelayMs(),
1017                          mDataLoaderManager->bindDelayMs());
1018      mDataLoaderManager->setDataLoaderStatusDestroyed();
1019  }
1020  
TEST_F(IncrementalServiceTest,testDataLoaderOnRestart)1021  TEST_F(IncrementalServiceTest, testDataLoaderOnRestart) {
1022      mIncFs->waitForPendingReadsSuccess();
1023      mIncFs->openMountSuccess();
1024  
1025      constexpr auto bindRetryInterval = 5s;
1026  
1027      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(11);
1028      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
1029      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(7);
1030      EXPECT_CALL(*mDataLoader, start(_)).Times(7);
1031      EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
1032      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1033      EXPECT_CALL(*mTimedQueue, addJob(_, _, _)).Times(4);
1034      TemporaryDir tempDir;
1035      int storageId =
1036              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1037                                                 IncrementalService::CreateOptions::CreateNew);
1038      ASSERT_GE(storageId, 0);
1039  
1040      // First binds to DataLoader fails... because it's restart.
1041      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1042              .WillByDefault(Invoke(mDataLoaderManager,
1043                                    &MockDataLoaderManager::bindToDataLoaderNotOkWithNoDelay));
1044  
1045      // Request DL start.
1046      mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {}, {}, {});
1047  
1048      // Retry callback present.
1049      ASSERT_EQ(storageId, mTimedQueue->mId);
1050      ASSERT_EQ(mTimedQueue->mAfter, bindRetryInterval);
1051      auto retryCallback = mTimedQueue->mWhat;
1052      mTimedQueue->clearJob(storageId);
1053  
1054      // Expecting the same bindToDataLoaderNotOkWithNoDelay call.
1055      mClock->advance(5s);
1056  
1057      retryCallback();
1058      // Retry callback present.
1059      ASSERT_EQ(storageId, mTimedQueue->mId);
1060      ASSERT_EQ(mTimedQueue->mAfter, bindRetryInterval);
1061      retryCallback = mTimedQueue->mWhat;
1062      mTimedQueue->clearJob(storageId);
1063  
1064      // Returning "binding" so that we can retry.
1065      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1066              .WillByDefault(Invoke(mDataLoaderManager,
1067                                    &MockDataLoaderManager::bindToDataLoaderBindingWithNoDelay));
1068  
1069      // Expecting bindToDataLoaderBindingWithNoDelay call.
1070      mClock->advance(5s);
1071  
1072      retryCallback();
1073      // No retry callback.
1074      ASSERT_EQ(mTimedQueue->mAfter, 0ms);
1075      ASSERT_EQ(mTimedQueue->mWhat, nullptr);
1076  
1077      // Should not change the bindToDataLoader call count
1078      ASSERT_NE(nullptr, mLooper->mCallback);
1079      ASSERT_NE(nullptr, mLooper->mCallbackData);
1080      auto looperCb = mLooper->mCallback;
1081      auto looperCbData = mLooper->mCallbackData;
1082      looperCb(-1, -1, looperCbData);
1083  
1084      // Expecting the same bindToDataLoaderBindingWithNoDelay call.
1085      mClock->advance(5s);
1086  
1087      // Use pending reads callback to trigger binding.
1088      looperCb(-1, -1, looperCbData);
1089  
1090      // No retry callback.
1091      ASSERT_EQ(mTimedQueue->mAfter, 0ms);
1092      ASSERT_EQ(mTimedQueue->mWhat, nullptr);
1093  
1094      // Now we are out of 10m "retry" budget, let's finally bind.
1095      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1096              .WillByDefault(Invoke(mDataLoaderManager, &MockDataLoaderManager::bindToDataLoaderOk));
1097      mClock->advance(11min);
1098  
1099      // Use pending reads callback to trigger binding.
1100      looperCb(-1, -1, looperCbData);
1101  
1102      // No retry callback.
1103      ASSERT_EQ(mTimedQueue->mAfter, 0ms);
1104      ASSERT_EQ(mTimedQueue->mWhat, nullptr);
1105  
1106      // And test the rest of the backoff.
1107      // Simulated crash/other connection breakage.
1108      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1109              .WillByDefault(Invoke(mDataLoaderManager,
1110                                    &MockDataLoaderManager::bindToDataLoaderOkWith1sDelay));
1111      mClock->advanceMs(mDataLoaderManager->bindDelayMs());
1112      mDataLoaderManager->setDataLoaderStatusDestroyed();
1113  
1114      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1115              .WillByDefault(Invoke(mDataLoaderManager,
1116                                    &MockDataLoaderManager::bindToDataLoaderOkWith10sDelay));
1117      mClock->advanceMs(mDataLoaderManager->bindDelayMs());
1118      mDataLoaderManager->setDataLoaderStatusDestroyed();
1119  
1120      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1121              .WillByDefault(Invoke(mDataLoaderManager,
1122                                    &MockDataLoaderManager::bindToDataLoaderOkWith100sDelay));
1123      mClock->advanceMs(mDataLoaderManager->bindDelayMs());
1124      mDataLoaderManager->setDataLoaderStatusDestroyed();
1125  
1126      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1127              .WillByDefault(Invoke(mDataLoaderManager,
1128                                    &MockDataLoaderManager::bindToDataLoaderOkWith1000sDelay));
1129      mClock->advanceMs(mDataLoaderManager->bindDelayMs());
1130      mDataLoaderManager->setDataLoaderStatusDestroyed();
1131  
1132      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1133              .WillByDefault(Invoke(mDataLoaderManager,
1134                                    &MockDataLoaderManager::bindToDataLoaderOkWith10000sDelay));
1135      mClock->advanceMs(mDataLoaderManager->bindDelayMs());
1136      mDataLoaderManager->setDataLoaderStatusDestroyed();
1137  
1138      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1139              .WillByDefault(Invoke(mDataLoaderManager,
1140                                    &MockDataLoaderManager::bindToDataLoaderOkWith10000sDelay));
1141      mClock->advanceMs(mDataLoaderManager->bindDelayMs());
1142      mDataLoaderManager->setDataLoaderStatusDestroyed();
1143  }
1144  
TEST_F(IncrementalServiceTest,testStartDataLoaderCreate)1145  TEST_F(IncrementalServiceTest, testStartDataLoaderCreate) {
1146      mDataLoader->initializeCreateOkNoStatus();
1147      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
1148      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
1149      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
1150      EXPECT_CALL(*mDataLoader, start(_)).Times(1);
1151      EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
1152      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1153      TemporaryDir tempDir;
1154      int storageId =
1155              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1156                                                 IncrementalService::CreateOptions::CreateNew);
1157      ASSERT_GE(storageId, 0);
1158      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1159                                                    {}, {}));
1160      mDataLoaderManager->setDataLoaderStatusCreated();
1161      mDataLoaderManager->setDataLoaderStatusStarted();
1162  }
1163  
TEST_F(IncrementalServiceTest,testStartDataLoaderPendingStart)1164  TEST_F(IncrementalServiceTest, testStartDataLoaderPendingStart) {
1165      mDataLoader->initializeCreateOkNoStatus();
1166      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
1167      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
1168      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
1169      EXPECT_CALL(*mDataLoader, start(_)).Times(1);
1170      EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
1171      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1172      TemporaryDir tempDir;
1173      int storageId =
1174              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1175                                                 IncrementalService::CreateOptions::CreateNew);
1176      ASSERT_GE(storageId, 0);
1177      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1178                                                    {}, {}));
1179      mDataLoaderManager->setDataLoaderStatusCreated();
1180  }
1181  
TEST_F(IncrementalServiceTest,testStartDataLoaderCreateUnavailable)1182  TEST_F(IncrementalServiceTest, testStartDataLoaderCreateUnavailable) {
1183      mDataLoader->initializeCreateOkNoStatus();
1184      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
1185      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
1186      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
1187      EXPECT_CALL(*mDataLoader, start(_)).Times(0);
1188      EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
1189      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1190      TemporaryDir tempDir;
1191      int storageId =
1192              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1193                                                 IncrementalService::CreateOptions::CreateNew);
1194      ASSERT_GE(storageId, 0);
1195      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1196                                                    {}, {}));
1197      mDataLoaderManager->setDataLoaderStatusUnavailable();
1198  }
1199  
TEST_F(IncrementalServiceTest,testStartDataLoaderCreateUnrecoverable)1200  TEST_F(IncrementalServiceTest, testStartDataLoaderCreateUnrecoverable) {
1201      mDataLoader->initializeCreateOkNoStatus();
1202      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
1203      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
1204      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
1205      EXPECT_CALL(*mDataLoader, start(_)).Times(0);
1206      EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
1207      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1208      TemporaryDir tempDir;
1209      int storageId =
1210              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1211                                                 IncrementalService::CreateOptions::CreateNew);
1212      ASSERT_GE(storageId, 0);
1213      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1214                                                    {}, {}));
1215      mDataLoaderManager->setDataLoaderStatusUnrecoverable();
1216  }
1217  
TEST_F(IncrementalServiceTest,testStartDataLoaderRecreateOnPendingReads)1218  TEST_F(IncrementalServiceTest, testStartDataLoaderRecreateOnPendingReads) {
1219      mIncFs->waitForPendingReadsSuccess();
1220      mIncFs->openMountSuccess();
1221      mDataLoader->initializeCreateOkNoStatus();
1222  
1223      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(2);
1224      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(2);
1225      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(2);
1226      EXPECT_CALL(*mDataLoader, start(_)).Times(0);
1227      EXPECT_CALL(*mDataLoader, destroy(_)).Times(2);
1228      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1229      EXPECT_CALL(*mLooper, addFd(MockIncFs::kPendingReadsFd, _, _, _, _)).Times(1);
1230      EXPECT_CALL(*mLooper, removeFd(MockIncFs::kPendingReadsFd)).Times(1);
1231      TemporaryDir tempDir;
1232      int storageId =
1233              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1234                                                 IncrementalService::CreateOptions::CreateNew);
1235      ASSERT_GE(storageId, 0);
1236      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1237                                                    {}, {}));
1238      mDataLoaderManager->setDataLoaderStatusUnrecoverable();
1239  
1240      // Timed callback present.
1241      ASSERT_EQ(storageId, mTimedQueue->mId);
1242      ASSERT_GE(mTimedQueue->mAfter, 10ms);
1243      auto timedCallback = mTimedQueue->mWhat;
1244      mTimedQueue->clearJob(storageId);
1245  
1246      // First callback call to propagate unrecoverable.
1247      timedCallback();
1248  
1249      // And second call to trigger recreation.
1250      ASSERT_NE(nullptr, mLooper->mCallback);
1251      ASSERT_NE(nullptr, mLooper->mCallbackData);
1252      mLooper->mCallback(-1, -1, mLooper->mCallbackData);
1253  }
1254  
TEST_F(IncrementalServiceTest,testStartDataLoaderUnavailable)1255  TEST_F(IncrementalServiceTest, testStartDataLoaderUnavailable) {
1256      mIncFs->openMountSuccess();
1257      mDataLoader->initializeCreateOkNoStatus();
1258  
1259      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(3);
1260      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(3);
1261      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(3);
1262      EXPECT_CALL(*mDataLoader, start(_)).Times(1);
1263      EXPECT_CALL(*mDataLoader, destroy(_)).Times(2);
1264      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1265      EXPECT_CALL(*mLooper, addFd(MockIncFs::kPendingReadsFd, _, _, _, _)).Times(1);
1266      EXPECT_CALL(*mLooper, removeFd(MockIncFs::kPendingReadsFd)).Times(1);
1267      TemporaryDir tempDir;
1268      int storageId =
1269              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1270                                                 IncrementalService::CreateOptions::CreateNew);
1271      ASSERT_GE(storageId, 0);
1272  
1273      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1274              .WillByDefault(Invoke(mDataLoaderManager,
1275                                    &MockDataLoaderManager::bindToDataLoaderOkWithNoDelay));
1276  
1277      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1278                                                    {}, {}));
1279  
1280      // Unavailable.
1281      mDataLoaderManager->setDataLoaderStatusUnavailable();
1282  
1283      // Timed callback present.
1284      ASSERT_EQ(storageId, mTimedQueue->mId);
1285      ASSERT_GE(mTimedQueue->mAfter, 10ms);
1286      auto timedCallback = mTimedQueue->mWhat;
1287      mTimedQueue->clearJob(storageId);
1288  
1289      // Propagating unavailable and expecting it to trigger rebind with 1s retry delay.
1290      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1291              .WillByDefault(Invoke(mDataLoaderManager,
1292                                    &MockDataLoaderManager::bindToDataLoaderOkWith1sDelay));
1293      timedCallback();
1294  
1295      // Unavailable #2.
1296      mDataLoaderManager->setDataLoaderStatusUnavailable();
1297  
1298      // Timed callback present.
1299      ASSERT_EQ(storageId, mTimedQueue->mId);
1300      ASSERT_GE(mTimedQueue->mAfter, 10ms);
1301      timedCallback = mTimedQueue->mWhat;
1302      mTimedQueue->clearJob(storageId);
1303  
1304      // Propagating unavailable and expecting it to trigger rebind with 10s retry delay.
1305      // This time succeed.
1306      mDataLoader->initializeCreateOk();
1307      ON_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _))
1308              .WillByDefault(Invoke(mDataLoaderManager,
1309                                    &MockDataLoaderManager::bindToDataLoaderOkWith10sDelay));
1310      timedCallback();
1311  }
1312  
TEST_F(IncrementalServiceTest,testStartDataLoaderUnhealthyStorage)1313  TEST_F(IncrementalServiceTest, testStartDataLoaderUnhealthyStorage) {
1314      mIncFs->openMountSuccess();
1315  
1316      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
1317      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
1318      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
1319      EXPECT_CALL(*mDataLoader, start(_)).Times(1);
1320      EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
1321      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1322      EXPECT_CALL(*mLooper, addFd(MockIncFs::kPendingReadsFd, _, _, _, _)).Times(2);
1323      EXPECT_CALL(*mLooper, removeFd(MockIncFs::kPendingReadsFd)).Times(2);
1324      EXPECT_CALL(*mTimedQueue, addJob(_, _, _)).Times(6);
1325  
1326      sp<NiceMock<MockStorageHealthListener>> listener{new NiceMock<MockStorageHealthListener>};
1327      NiceMock<MockStorageHealthListener>* listenerMock = listener.get();
1328      EXPECT_CALL(*listenerMock, onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_OK))
1329              .Times(2);
1330      EXPECT_CALL(*listenerMock,
1331                  onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_READS_PENDING))
1332              .Times(1);
1333      EXPECT_CALL(*listenerMock, onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_BLOCKED))
1334              .Times(1);
1335      EXPECT_CALL(*listenerMock, onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_UNHEALTHY))
1336              .Times(2);
1337  
1338      StorageHealthCheckParams params;
1339      params.blockedTimeoutMs = 10000;
1340      params.unhealthyTimeoutMs = 20000;
1341      params.unhealthyMonitoringMs = 30000;
1342  
1343      using MS = std::chrono::milliseconds;
1344      using MCS = std::chrono::microseconds;
1345  
1346      const auto blockedTimeout = MS(params.blockedTimeoutMs);
1347      const auto unhealthyTimeout = MS(params.unhealthyTimeoutMs);
1348      const auto unhealthyMonitoring = MS(params.unhealthyMonitoringMs);
1349  
1350      const uint64_t kFirstTimestampUs = 1000000000ll;
1351      const uint64_t kBlockedTimestampUs =
1352              kFirstTimestampUs - std::chrono::duration_cast<MCS>(blockedTimeout).count();
1353      const uint64_t kUnhealthyTimestampUs =
1354              kFirstTimestampUs - std::chrono::duration_cast<MCS>(unhealthyTimeout).count();
1355  
1356      TemporaryDir tempDir;
1357      int storageId =
1358              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1359                                                 IncrementalService::CreateOptions::CreateNew);
1360      ASSERT_GE(storageId, 0);
1361      mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {},
1362                                        std::move(params), listener, {});
1363  
1364      // Healthy state, registered for pending reads.
1365      ASSERT_NE(nullptr, mLooper->mCallback);
1366      ASSERT_NE(nullptr, mLooper->mCallbackData);
1367      ASSERT_EQ(storageId, listener->mStorageId);
1368      ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_OK, listener->mStatus);
1369      checkHealthMetrics(storageId, 0, listener->mStatus);
1370  
1371      // Looper/epoll callback.
1372      mIncFs->waitForPendingReadsSuccess(kFirstTimestampUs);
1373      mLooper->mCallback(-1, -1, mLooper->mCallbackData);
1374  
1375      // Unregister from pending reads and wait.
1376      ASSERT_EQ(nullptr, mLooper->mCallback);
1377      ASSERT_EQ(nullptr, mLooper->mCallbackData);
1378      ASSERT_EQ(storageId, listener->mStorageId);
1379      ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_READS_PENDING, listener->mStatus);
1380      // Timed callback present.
1381      ASSERT_EQ(storageId, mTimedQueue->mId);
1382      ASSERT_GE(mTimedQueue->mAfter, blockedTimeout);
1383      auto timedCallback = mTimedQueue->mWhat;
1384      mTimedQueue->clearJob(storageId);
1385  
1386      // Timed job callback for blocked.
1387      mIncFs->waitForPendingReadsSuccess(kBlockedTimestampUs);
1388      timedCallback();
1389  
1390      // Still not registered, and blocked.
1391      ASSERT_EQ(nullptr, mLooper->mCallback);
1392      ASSERT_EQ(nullptr, mLooper->mCallbackData);
1393      ASSERT_EQ(storageId, listener->mStorageId);
1394      ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_BLOCKED, listener->mStatus);
1395      checkHealthMetrics(storageId, params.blockedTimeoutMs, listener->mStatus);
1396  
1397      // Timed callback present.
1398      ASSERT_EQ(storageId, mTimedQueue->mId);
1399      ASSERT_GE(mTimedQueue->mAfter, 1000ms);
1400      timedCallback = mTimedQueue->mWhat;
1401      mTimedQueue->clearJob(storageId);
1402  
1403      // Timed job callback for unhealthy.
1404      mIncFs->waitForPendingReadsSuccess(kUnhealthyTimestampUs);
1405      timedCallback();
1406  
1407      // Still not registered, and blocked.
1408      ASSERT_EQ(nullptr, mLooper->mCallback);
1409      ASSERT_EQ(nullptr, mLooper->mCallbackData);
1410      ASSERT_EQ(storageId, listener->mStorageId);
1411      ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_UNHEALTHY, listener->mStatus);
1412      checkHealthMetrics(storageId, params.unhealthyTimeoutMs, listener->mStatus);
1413  
1414      // Timed callback present.
1415      ASSERT_EQ(storageId, mTimedQueue->mId);
1416      ASSERT_GE(mTimedQueue->mAfter, unhealthyMonitoring);
1417      timedCallback = mTimedQueue->mWhat;
1418      mTimedQueue->clearJob(storageId);
1419  
1420      // One more unhealthy.
1421      mIncFs->waitForPendingReadsSuccess(kUnhealthyTimestampUs);
1422      timedCallback();
1423  
1424      // Still not registered, and blocked.
1425      ASSERT_EQ(nullptr, mLooper->mCallback);
1426      ASSERT_EQ(nullptr, mLooper->mCallbackData);
1427      ASSERT_EQ(storageId, listener->mStorageId);
1428      ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_UNHEALTHY, listener->mStatus);
1429      checkHealthMetrics(storageId, params.unhealthyTimeoutMs, listener->mStatus);
1430  
1431      // Timed callback present.
1432      ASSERT_EQ(storageId, mTimedQueue->mId);
1433      ASSERT_GE(mTimedQueue->mAfter, unhealthyMonitoring);
1434      timedCallback = mTimedQueue->mWhat;
1435      mTimedQueue->clearJob(storageId);
1436  
1437      // And now healthy.
1438      mIncFs->waitForPendingReadsTimeout();
1439      timedCallback();
1440  
1441      // Healthy state, registered for pending reads.
1442      ASSERT_NE(nullptr, mLooper->mCallback);
1443      ASSERT_NE(nullptr, mLooper->mCallbackData);
1444      ASSERT_EQ(storageId, listener->mStorageId);
1445      ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_OK, listener->mStatus);
1446      checkHealthMetrics(storageId, 0, listener->mStatus);
1447  }
1448  
TEST_F(IncrementalServiceTest,testSetIncFsMountOptionsSuccess)1449  TEST_F(IncrementalServiceTest, testSetIncFsMountOptionsSuccess) {
1450      mVold->setIncFsMountOptionsSuccess();
1451      mAppOpsManager->checkPermissionSuccess();
1452  
1453      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_));
1454      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1455      // on startLoading
1456      EXPECT_CALL(*mVold, setIncFsMountOptions(_, false, _, _)).Times(1);
1457      // We are calling setIncFsMountOptions(true).
1458      EXPECT_CALL(*mVold, setIncFsMountOptions(_, true, _, _)).Times(1);
1459      // After setIncFsMountOptions succeeded expecting to start watching.
1460      EXPECT_CALL(*mAppOpsManager, startWatchingMode(_, _, _)).Times(1);
1461      // Not expecting callback removal.
1462      EXPECT_CALL(*mAppOpsManager, stopWatchingMode(_)).Times(0);
1463      TemporaryDir tempDir;
1464      int storageId =
1465              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1466                                                 IncrementalService::CreateOptions::CreateNew);
1467      ASSERT_GE(storageId, 0);
1468      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1469                                                    {}, {}));
1470      ASSERT_GE(mDataLoader->setStorageParams(true), 0);
1471  }
1472  
TEST_F(IncrementalServiceTest,testSetIncFsMountOptionsSuccessAndDisabled)1473  TEST_F(IncrementalServiceTest, testSetIncFsMountOptionsSuccessAndDisabled) {
1474      mVold->setIncFsMountOptionsSuccess();
1475      mAppOpsManager->checkPermissionSuccess();
1476  
1477      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_));
1478      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1479      // Enabling and then disabling readlogs.
1480      EXPECT_CALL(*mVold, setIncFsMountOptions(_, true, _, _)).Times(1);
1481      EXPECT_CALL(*mVold, setIncFsMountOptions(_, false, _, _)).Times(2);
1482      // After setIncFsMountOptions succeeded expecting to start watching.
1483      EXPECT_CALL(*mAppOpsManager, startWatchingMode(_, _, _)).Times(1);
1484      // Not expecting callback removal.
1485      EXPECT_CALL(*mAppOpsManager, stopWatchingMode(_)).Times(0);
1486      TemporaryDir tempDir;
1487      int storageId =
1488              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1489                                                 IncrementalService::CreateOptions::CreateNew);
1490      ASSERT_GE(storageId, 0);
1491      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1492                                                    {}, {}));
1493      ASSERT_GE(mDataLoader->setStorageParams(true), 0);
1494      // Now disable.
1495      mIncrementalService->disallowReadLogs(storageId);
1496      ASSERT_EQ(mDataLoader->setStorageParams(true), -EPERM);
1497  }
1498  
TEST_F(IncrementalServiceTest,testSetIncFsMountOptionsSuccessAndTimedOut)1499  TEST_F(IncrementalServiceTest, testSetIncFsMountOptionsSuccessAndTimedOut) {
1500      mVold->setIncFsMountOptionsSuccess();
1501      mAppOpsManager->checkPermissionSuccess();
1502  
1503      const auto readLogsMaxInterval = 2h;
1504  
1505      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_));
1506      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1507      // Enabling and then disabling readlogs.
1508      EXPECT_CALL(*mVold, setIncFsMountOptions(_, true, _, _)).Times(2);
1509      EXPECT_CALL(*mVold, setIncFsMountOptions(_, false, _, _)).Times(2);
1510      // After setIncFsMountOptions succeeded expecting to start watching.
1511      EXPECT_CALL(*mAppOpsManager, startWatchingMode(_, _, _)).Times(1);
1512      // Not expecting callback removal.
1513      EXPECT_CALL(*mAppOpsManager, stopWatchingMode(_)).Times(0);
1514      EXPECT_CALL(*mTimedQueue, addJob(_, _, _)).Times(2);
1515      TemporaryDir tempDir;
1516      int storageId =
1517              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1518                                                 IncrementalService::CreateOptions::CreateNew);
1519      ASSERT_GE(storageId, 0);
1520      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1521                                                    {}, {}));
1522  
1523      // Disable readlogs callback present.
1524      ASSERT_EQ(storageId, mTimedQueue->mId);
1525      ASSERT_EQ(mTimedQueue->mAfter, readLogsMaxInterval);
1526      auto callback = mTimedQueue->mWhat;
1527      mTimedQueue->clearJob(storageId);
1528  
1529      ASSERT_GE(mDataLoader->setStorageParams(true), 0);
1530      // Now advance clock for 1hr.
1531      mClock->advance(1h);
1532      ASSERT_GE(mDataLoader->setStorageParams(true), 0);
1533      // Now call the timed callback, it should turn off the readlogs.
1534      callback();
1535      // Now advance clock for 2hrs.
1536      mClock->advance(readLogsMaxInterval);
1537      ASSERT_EQ(mDataLoader->setStorageParams(true), -EPERM);
1538  }
1539  
TEST_F(IncrementalServiceTest,testSetIncFsMountOptionsSuccessAndNoTimedOutForSystem)1540  TEST_F(IncrementalServiceTest, testSetIncFsMountOptionsSuccessAndNoTimedOutForSystem) {
1541      mVold->setIncFsMountOptionsSuccess();
1542      mAppOpsManager->checkPermissionSuccess();
1543  
1544      const auto readLogsMaxInterval = 2h;
1545  
1546      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_));
1547      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1548      // Enabling and then disabling readlogs.
1549      EXPECT_CALL(*mVold, setIncFsMountOptions(_, true, _, _)).Times(3);
1550      EXPECT_CALL(*mVold, setIncFsMountOptions(_, false, _, _)).Times(1);
1551      // After setIncFsMountOptions succeeded expecting to start watching.
1552      EXPECT_CALL(*mAppOpsManager, startWatchingMode(_, _, _)).Times(1);
1553      // Not expecting callback removal.
1554      EXPECT_CALL(*mAppOpsManager, stopWatchingMode(_)).Times(0);
1555      EXPECT_CALL(*mTimedQueue, addJob(_, _, _)).Times(2);
1556      // System data loader.
1557      mDataLoaderParcel.packageName = "android";
1558      TemporaryDir tempDir;
1559      int storageId =
1560              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1561                                                 IncrementalService::CreateOptions::CreateNew);
1562      ASSERT_GE(storageId, 0);
1563      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1564                                                    {}, {}));
1565  
1566      // IfsState callback.
1567      auto callback = mTimedQueue->mWhat;
1568      mTimedQueue->clearJob(storageId);
1569  
1570      ASSERT_GE(mDataLoader->setStorageParams(true), 0);
1571      // Now advance clock for 1hr.
1572      mClock->advance(1h);
1573      ASSERT_GE(mDataLoader->setStorageParams(true), 0);
1574      // Now advance clock for 2hrs.
1575      mClock->advance(readLogsMaxInterval);
1576      // IfsStorage callback should not affect anything.
1577      callback();
1578      ASSERT_EQ(mDataLoader->setStorageParams(true), 0);
1579  }
1580  
TEST_F(IncrementalServiceTest,testSetIncFsMountOptionsSuccessAndNewInstall)1581  TEST_F(IncrementalServiceTest, testSetIncFsMountOptionsSuccessAndNewInstall) {
1582      mVold->setIncFsMountOptionsSuccess();
1583      mAppOpsManager->checkPermissionSuccess();
1584  
1585      const auto readLogsMaxInterval = 2h;
1586  
1587      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(2);
1588      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1589      // Enabling and then disabling readlogs.
1590      EXPECT_CALL(*mVold, setIncFsMountOptions(_, true, _, _)).Times(5);
1591      EXPECT_CALL(*mVold, setIncFsMountOptions(_, false, _, _)).Times(3);
1592      // After setIncFsMountOptions succeeded expecting to start watching.
1593      EXPECT_CALL(*mAppOpsManager, startWatchingMode(_, _, _)).Times(1);
1594      // Not expecting callback removal.
1595      EXPECT_CALL(*mAppOpsManager, stopWatchingMode(_)).Times(0);
1596      EXPECT_CALL(*mTimedQueue, addJob(_, _, _)).Times(4);
1597      TemporaryDir tempDir;
1598      int storageId =
1599              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1600                                                 IncrementalService::CreateOptions::CreateNew);
1601      ASSERT_GE(storageId, 0);
1602  
1603      // Before install - long timeouts.
1604      ASSERT_TRUE(mVold->readTimeoutsEnabled());
1605  
1606      auto dataLoaderParcel = mDataLoaderParcel;
1607      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(dataLoaderParcel), {}, {},
1608                                                    {}, {}));
1609      // During install - short timeouts.
1610      ASSERT_FALSE(mVold->readTimeoutsEnabled());
1611  
1612      // Disable readlogs callback present.
1613      ASSERT_EQ(storageId, mTimedQueue->mId);
1614      ASSERT_EQ(mTimedQueue->mAfter, readLogsMaxInterval);
1615      auto callback = mTimedQueue->mWhat;
1616      mTimedQueue->clearJob(storageId);
1617  
1618      ASSERT_GE(mDataLoader->setStorageParams(true), 0);
1619      // Now advance clock for 1.5hrs.
1620      mClock->advance(90min);
1621      ASSERT_GE(mDataLoader->setStorageParams(true), 0);
1622  
1623      mIncrementalService->onInstallationComplete(storageId);
1624      // After install - long timeouts.
1625      ASSERT_TRUE(mVold->readTimeoutsEnabled());
1626  
1627      // New installation.
1628      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1629                                                    {}, {}));
1630      // New installation - short timeouts.
1631      ASSERT_FALSE(mVold->readTimeoutsEnabled());
1632  
1633      // New callback present.
1634      ASSERT_EQ(storageId, mTimedQueue->mId);
1635      ASSERT_EQ(mTimedQueue->mAfter, readLogsMaxInterval);
1636      auto callback2 = mTimedQueue->mWhat;
1637      mTimedQueue->clearJob(storageId);
1638  
1639      // Old callback should not disable readlogs (setIncFsMountOptions should be called only once).
1640      callback();
1641      // Advance clock for another 1.5hrs.
1642      mClock->advance(90min);
1643      // Still success even it's 3hrs past first install.
1644      ASSERT_GE(mDataLoader->setStorageParams(true), 0);
1645  
1646      // New one should disable.
1647      callback2();
1648      // And timeout.
1649      mClock->advance(90min);
1650      ASSERT_EQ(mDataLoader->setStorageParams(true), -EPERM);
1651  
1652      mIncrementalService->onInstallationComplete(storageId);
1653      // After install - long timeouts.
1654      ASSERT_TRUE(mVold->readTimeoutsEnabled());
1655  }
1656  
TEST_F(IncrementalServiceTest,testSetIncFsMountOptionsSuccessAndPermissionChanged)1657  TEST_F(IncrementalServiceTest, testSetIncFsMountOptionsSuccessAndPermissionChanged) {
1658      mVold->setIncFsMountOptionsSuccess();
1659      mAppOpsManager->checkPermissionSuccess();
1660      mAppOpsManager->initializeStartWatchingMode();
1661  
1662      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_));
1663      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1664      // We are calling setIncFsMountOptions(true).
1665      EXPECT_CALL(*mVold, setIncFsMountOptions(_, true, _, _)).Times(1);
1666      // setIncFsMountOptions(false) is called on the callback.
1667      EXPECT_CALL(*mVold, setIncFsMountOptions(_, false, _, _)).Times(2);
1668      // After setIncFsMountOptions succeeded expecting to start watching.
1669      EXPECT_CALL(*mAppOpsManager, startWatchingMode(_, _, _)).Times(1);
1670      // After callback is called, disable read logs and remove callback.
1671      EXPECT_CALL(*mAppOpsManager, stopWatchingMode(_)).Times(1);
1672      TemporaryDir tempDir;
1673      int storageId =
1674              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1675                                                 IncrementalService::CreateOptions::CreateNew);
1676      ASSERT_GE(storageId, 0);
1677      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1678                                                    {}, {}));
1679      ASSERT_GE(mDataLoader->setStorageParams(true), 0);
1680      ASSERT_NE(nullptr, mAppOpsManager->mStoredCallback.get());
1681      mAppOpsManager->mStoredCallback->opChanged(0, {});
1682  }
1683  
TEST_F(IncrementalServiceTest,testSetIncFsMountOptionsCheckPermissionFails)1684  TEST_F(IncrementalServiceTest, testSetIncFsMountOptionsCheckPermissionFails) {
1685      mAppOpsManager->checkPermissionFails();
1686  
1687      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_));
1688      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1689      // checkPermission fails, no calls to set opitions,  start or stop WatchingMode.
1690      EXPECT_CALL(*mVold, setIncFsMountOptions(_, true, _, _)).Times(0);
1691      EXPECT_CALL(*mVold, setIncFsMountOptions(_, false, _, _)).Times(1);
1692      EXPECT_CALL(*mAppOpsManager, startWatchingMode(_, _, _)).Times(0);
1693      EXPECT_CALL(*mAppOpsManager, stopWatchingMode(_)).Times(0);
1694      TemporaryDir tempDir;
1695      int storageId =
1696              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1697                                                 IncrementalService::CreateOptions::CreateNew);
1698      ASSERT_GE(storageId, 0);
1699      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1700                                                    {}, {}));
1701      ASSERT_LT(mDataLoader->setStorageParams(true), 0);
1702  }
1703  
TEST_F(IncrementalServiceTest,testSetIncFsMountOptionsCheckPermissionNoCrossUsers)1704  TEST_F(IncrementalServiceTest, testSetIncFsMountOptionsCheckPermissionNoCrossUsers) {
1705      mAppOpsManager->checkPermissionNoCrossUsers();
1706  
1707      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_));
1708      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1709      // checkPermission fails, no calls to set opitions,  start or stop WatchingMode.
1710      EXPECT_CALL(*mVold, setIncFsMountOptions(_, true, _, _)).Times(0);
1711      EXPECT_CALL(*mVold, setIncFsMountOptions(_, false, _, _)).Times(1);
1712      EXPECT_CALL(*mAppOpsManager, startWatchingMode(_, _, _)).Times(0);
1713      EXPECT_CALL(*mAppOpsManager, stopWatchingMode(_)).Times(0);
1714      TemporaryDir tempDir;
1715      int storageId =
1716              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1717                                                 IncrementalService::CreateOptions::CreateNew);
1718      ASSERT_GE(storageId, 0);
1719      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1720                                                    {}, {}));
1721      ASSERT_LT(mDataLoader->setStorageParams(true), 0);
1722  }
1723  
TEST_F(IncrementalServiceTest,testSetIncFsMountOptionsFails)1724  TEST_F(IncrementalServiceTest, testSetIncFsMountOptionsFails) {
1725      mVold->setIncFsMountOptionsFails();
1726      mAppOpsManager->checkPermissionSuccess();
1727  
1728      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_));
1729      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1730      // We are calling setIncFsMountOptions.
1731      EXPECT_CALL(*mVold, setIncFsMountOptions(_, true, _, _)).Times(1);
1732      EXPECT_CALL(*mVold, setIncFsMountOptions(_, false, _, _)).Times(1);
1733      // setIncFsMountOptions fails, no calls to start or stop WatchingMode.
1734      EXPECT_CALL(*mAppOpsManager, startWatchingMode(_, _, _)).Times(0);
1735      EXPECT_CALL(*mAppOpsManager, stopWatchingMode(_)).Times(0);
1736      TemporaryDir tempDir;
1737      int storageId =
1738              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1739                                                 IncrementalService::CreateOptions::CreateNew);
1740      ASSERT_GE(storageId, 0);
1741      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1742                                                    {}, {}));
1743      ASSERT_LT(mDataLoader->setStorageParams(true), 0);
1744  }
1745  
TEST_F(IncrementalServiceTest,testMakeDirectory)1746  TEST_F(IncrementalServiceTest, testMakeDirectory) {
1747      TemporaryDir tempDir;
1748      int storageId =
1749              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1750                                                 IncrementalService::CreateOptions::CreateNew);
1751      std::string dir_path("test");
1752  
1753      // Expecting incfs to call makeDir on a path like:
1754      // <root>/*/mount/<storage>/test
1755      EXPECT_CALL(*mIncFs,
1756                  makeDir(_, Truly([&](std::string_view arg) {
1757                              return arg.starts_with(mRootDir.path) &&
1758                                      arg.ends_with("/mount/st_1_0/" + dir_path);
1759                          }),
1760                          _));
1761      auto res = mIncrementalService->makeDir(storageId, dir_path, 0555);
1762      ASSERT_EQ(res, 0);
1763  }
1764  
TEST_F(IncrementalServiceTest,testMakeDirectories)1765  TEST_F(IncrementalServiceTest, testMakeDirectories) {
1766      TemporaryDir tempDir;
1767      int storageId =
1768              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1769                                                 IncrementalService::CreateOptions::CreateNew);
1770      auto first = "first"sv;
1771      auto second = "second"sv;
1772      auto third = "third"sv;
1773      auto dir_path = std::string(first) + "/" + std::string(second) + "/" + std::string(third);
1774  
1775      EXPECT_CALL(*mIncFs,
1776                  makeDirs(_, Truly([&](std::string_view arg) {
1777                               return arg.starts_with(mRootDir.path) &&
1778                                       arg.ends_with("/mount/st_1_0/" + dir_path);
1779                           }),
1780                           _));
1781      auto res = mIncrementalService->makeDirs(storageId, dir_path, 0555);
1782      ASSERT_EQ(res, 0);
1783  }
1784  
TEST_F(IncrementalServiceTest,testIsFileFullyLoadedNoData)1785  TEST_F(IncrementalServiceTest, testIsFileFullyLoadedNoData) {
1786      TemporaryDir tempDir;
1787      int storageId =
1788              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1789                                                 IncrementalService::CreateOptions::CreateNew);
1790      EXPECT_CALL(*mIncFs, isFileFullyLoaded(_, An<std::string_view>()))
1791              .Times(1)
1792              .WillOnce(Return(incfs::LoadingState::MissingBlocks));
1793      ASSERT_GT((int)mIncrementalService->isFileFullyLoaded(storageId, "base.apk"), 0);
1794  }
1795  
TEST_F(IncrementalServiceTest,testIsFileFullyLoadedError)1796  TEST_F(IncrementalServiceTest, testIsFileFullyLoadedError) {
1797      TemporaryDir tempDir;
1798      int storageId =
1799              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1800                                                 IncrementalService::CreateOptions::CreateNew);
1801      EXPECT_CALL(*mIncFs, isFileFullyLoaded(_, An<std::string_view>()))
1802              .Times(1)
1803              .WillOnce(Return(incfs::LoadingState(-1)));
1804      ASSERT_LT((int)mIncrementalService->isFileFullyLoaded(storageId, "base.apk"), 0);
1805  }
1806  
TEST_F(IncrementalServiceTest,testIsFileFullyLoadedSuccess)1807  TEST_F(IncrementalServiceTest, testIsFileFullyLoadedSuccess) {
1808      TemporaryDir tempDir;
1809      int storageId =
1810              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1811                                                 IncrementalService::CreateOptions::CreateNew);
1812      EXPECT_CALL(*mIncFs, isFileFullyLoaded(_, An<std::string_view>()))
1813              .Times(1)
1814              .WillOnce(Return(incfs::LoadingState::Full));
1815      ASSERT_EQ(0, (int)mIncrementalService->isFileFullyLoaded(storageId, "base.apk"));
1816  }
1817  
TEST_F(IncrementalServiceTest,testGetLoadingProgressSuccessWithNoFile)1818  TEST_F(IncrementalServiceTest, testGetLoadingProgressSuccessWithNoFile) {
1819      mIncFs->countFilledBlocksSuccess();
1820      mFs->hasNoFile();
1821  
1822      TemporaryDir tempDir;
1823      int storageId =
1824              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1825                                                 IncrementalService::CreateOptions::CreateNew);
1826      ASSERT_EQ(1, mIncrementalService->getLoadingProgress(storageId).getProgress());
1827  }
1828  
TEST_F(IncrementalServiceTest,testGetLoadingProgressFailsWithFailedRanges)1829  TEST_F(IncrementalServiceTest, testGetLoadingProgressFailsWithFailedRanges) {
1830      mIncFs->countFilledBlocksFails();
1831      mFs->hasFiles();
1832  
1833      TemporaryDir tempDir;
1834      int storageId =
1835              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1836                                                 IncrementalService::CreateOptions::CreateNew);
1837      EXPECT_CALL(*mIncFs, countFilledBlocks(_, _)).Times(1);
1838      ASSERT_EQ(-1, mIncrementalService->getLoadingProgress(storageId).getProgress());
1839  }
1840  
TEST_F(IncrementalServiceTest,testGetLoadingProgressSuccessWithEmptyRanges)1841  TEST_F(IncrementalServiceTest, testGetLoadingProgressSuccessWithEmptyRanges) {
1842      mIncFs->countFilledBlocksEmpty();
1843      mFs->hasFiles();
1844  
1845      TemporaryDir tempDir;
1846      int storageId =
1847              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1848                                                 IncrementalService::CreateOptions::CreateNew);
1849      EXPECT_CALL(*mIncFs, countFilledBlocks(_, _)).Times(3);
1850      ASSERT_EQ(1, mIncrementalService->getLoadingProgress(storageId).getProgress());
1851  }
1852  
TEST_F(IncrementalServiceTest,testGetLoadingProgressSuccess)1853  TEST_F(IncrementalServiceTest, testGetLoadingProgressSuccess) {
1854      mIncFs->countFilledBlocksSuccess();
1855      mFs->hasFiles();
1856  
1857      TemporaryDir tempDir;
1858      int storageId =
1859              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1860                                                 IncrementalService::CreateOptions::CreateNew);
1861      EXPECT_CALL(*mIncFs, countFilledBlocks(_, _)).Times(3);
1862      ASSERT_EQ(0.5, mIncrementalService->getLoadingProgress(storageId).getProgress());
1863  }
1864  
TEST_F(IncrementalServiceTest,testRegisterLoadingProgressListenerSuccess)1865  TEST_F(IncrementalServiceTest, testRegisterLoadingProgressListenerSuccess) {
1866      mIncFs->countFilledBlocksSuccess();
1867      mFs->hasFiles();
1868  
1869      TemporaryDir tempDir;
1870      int storageId =
1871              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1872                                                 IncrementalService::CreateOptions::CreateNew);
1873      sp<NiceMock<MockStorageLoadingProgressListener>> listener{
1874              new NiceMock<MockStorageLoadingProgressListener>};
1875      NiceMock<MockStorageLoadingProgressListener>* listenerMock = listener.get();
1876      EXPECT_CALL(*listenerMock, onStorageLoadingProgressChanged(_, _)).Times(2);
1877      EXPECT_CALL(*mProgressUpdateJobQueue, addJob(_, _, _)).Times(2);
1878      mIncrementalService->registerLoadingProgressListener(storageId, listener);
1879      // Timed callback present.
1880      ASSERT_EQ(storageId, mProgressUpdateJobQueue->mId);
1881      ASSERT_EQ(mProgressUpdateJobQueue->mAfter, 1000ms);
1882      auto timedCallback = mProgressUpdateJobQueue->mWhat;
1883      timedCallback();
1884      ASSERT_EQ(storageId, mProgressUpdateJobQueue->mId);
1885      ASSERT_EQ(mProgressUpdateJobQueue->mAfter, 1000ms);
1886      mIncrementalService->unregisterLoadingProgressListener(storageId);
1887      ASSERT_EQ(mProgressUpdateJobQueue->mAfter, Milliseconds{});
1888  }
1889  
TEST_F(IncrementalServiceTest,testRegisterLoadingProgressListenerFailsToGetProgress)1890  TEST_F(IncrementalServiceTest, testRegisterLoadingProgressListenerFailsToGetProgress) {
1891      mIncFs->countFilledBlocksFails();
1892      mFs->hasFiles();
1893  
1894      TemporaryDir tempDir;
1895      int storageId =
1896              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1897                                                 IncrementalService::CreateOptions::CreateNew);
1898      sp<NiceMock<MockStorageLoadingProgressListener>> listener{
1899              new NiceMock<MockStorageLoadingProgressListener>};
1900      NiceMock<MockStorageLoadingProgressListener>* listenerMock = listener.get();
1901      EXPECT_CALL(*listenerMock, onStorageLoadingProgressChanged(_, _)).Times(0);
1902      mIncrementalService->registerLoadingProgressListener(storageId, listener);
1903  }
1904  
TEST_F(IncrementalServiceTest,testStartDataLoaderUnbindOnAllDone)1905  TEST_F(IncrementalServiceTest, testStartDataLoaderUnbindOnAllDone) {
1906      mFs->hasFiles();
1907  
1908      const auto stateUpdateInterval = 1s;
1909  
1910      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
1911      // No unbinding just yet.
1912      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
1913      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
1914      EXPECT_CALL(*mDataLoader, start(_)).Times(1);
1915      EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
1916      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1917      // System data loader to get rid of readlog timeout callback.
1918      mDataLoaderParcel.packageName = "android";
1919      TemporaryDir tempDir;
1920      int storageId =
1921              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1922                                                 IncrementalService::CreateOptions::CreateNew);
1923      ASSERT_GE(storageId, 0);
1924      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1925                                                    {}, {}));
1926  
1927      // Started.
1928      ASSERT_EQ(mDataLoader->status(), IDataLoaderStatusListener::DATA_LOADER_STARTED);
1929  
1930      // IfsState callback present.
1931      ASSERT_EQ(IncrementalService::kAllStoragesId, mTimedQueue->mId);
1932      ASSERT_EQ(mTimedQueue->mAfter, stateUpdateInterval);
1933      auto callback = mTimedQueue->mWhat;
1934      mTimedQueue->clearJob(IncrementalService::kAllStoragesId);
1935  
1936      // Not loaded yet.
1937      EXPECT_CALL(*mIncFs, isEverythingFullyLoaded(_))
1938              .WillOnce(Return(incfs::LoadingState::MissingBlocks));
1939  
1940      // Send the callback, should not do anything.
1941      callback();
1942  
1943      // Still started.
1944      ASSERT_EQ(mDataLoader->status(), IDataLoaderStatusListener::DATA_LOADER_STARTED);
1945  
1946      // Still present.
1947      ASSERT_EQ(IncrementalService::kAllStoragesId, mTimedQueue->mId);
1948      ASSERT_EQ(mTimedQueue->mAfter, stateUpdateInterval);
1949      callback = mTimedQueue->mWhat;
1950      mTimedQueue->clearJob(IncrementalService::kAllStoragesId);
1951  
1952      // Fully loaded.
1953      EXPECT_CALL(*mIncFs, isEverythingFullyLoaded(_)).WillOnce(Return(incfs::LoadingState::Full));
1954      // Expect the unbind.
1955      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
1956  
1957      callback();
1958  
1959      // Destroyed.
1960      ASSERT_EQ(mDataLoader->status(), IDataLoaderStatusListener::DATA_LOADER_DESTROYED);
1961  }
1962  
TEST_F(IncrementalServiceTest,testStartDataLoaderUnbindOnAllDoneWithReadlogs)1963  TEST_F(IncrementalServiceTest, testStartDataLoaderUnbindOnAllDoneWithReadlogs) {
1964      mFs->hasFiles();
1965  
1966      // Readlogs.
1967      mVold->setIncFsMountOptionsSuccess();
1968      mAppOpsManager->checkPermissionSuccess();
1969  
1970      const auto stateUpdateInterval = 1s;
1971  
1972      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
1973      // No unbinding just yet.
1974      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(0);
1975      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
1976      EXPECT_CALL(*mDataLoader, start(_)).Times(1);
1977      EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
1978      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
1979      // System data loader to get rid of readlog timeout callback.
1980      mDataLoaderParcel.packageName = "android";
1981      TemporaryDir tempDir;
1982      int storageId =
1983              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
1984                                                 IncrementalService::CreateOptions::CreateNew);
1985      ASSERT_GE(storageId, 0);
1986      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
1987                                                    {}, {}));
1988  
1989      // Started.
1990      ASSERT_EQ(mDataLoader->status(), IDataLoaderStatusListener::DATA_LOADER_STARTED);
1991  
1992      // IfsState callback present.
1993      ASSERT_EQ(IncrementalService::kAllStoragesId, mTimedQueue->mId);
1994      ASSERT_EQ(mTimedQueue->mAfter, stateUpdateInterval);
1995      auto callback = mTimedQueue->mWhat;
1996      mTimedQueue->clearJob(IncrementalService::kAllStoragesId);
1997  
1998      // Not loaded yet.
1999      EXPECT_CALL(*mIncFs, isEverythingFullyLoaded(_))
2000              .WillOnce(Return(incfs::LoadingState::MissingBlocks));
2001  
2002      // Send the callback, should not do anything.
2003      callback();
2004  
2005      // Still started.
2006      ASSERT_EQ(mDataLoader->status(), IDataLoaderStatusListener::DATA_LOADER_STARTED);
2007  
2008      // Still present.
2009      ASSERT_EQ(IncrementalService::kAllStoragesId, mTimedQueue->mId);
2010      ASSERT_EQ(mTimedQueue->mAfter, stateUpdateInterval);
2011      callback = mTimedQueue->mWhat;
2012      mTimedQueue->clearJob(IncrementalService::kAllStoragesId);
2013  
2014      // Fully loaded.
2015      EXPECT_CALL(*mIncFs, isEverythingFullyLoaded(_))
2016              .WillOnce(Return(incfs::LoadingState::Full))
2017              .WillOnce(Return(incfs::LoadingState::Full));
2018      // But with readlogs.
2019      ASSERT_GE(mDataLoader->setStorageParams(true), 0);
2020  
2021      // Send the callback, still nothing.
2022      callback();
2023  
2024      // Still started.
2025      ASSERT_EQ(mDataLoader->status(), IDataLoaderStatusListener::DATA_LOADER_STARTED);
2026  
2027      // Still present.
2028      ASSERT_EQ(IncrementalService::kAllStoragesId, mTimedQueue->mId);
2029      ASSERT_EQ(mTimedQueue->mAfter, stateUpdateInterval);
2030      callback = mTimedQueue->mWhat;
2031      mTimedQueue->clearJob(IncrementalService::kAllStoragesId);
2032  
2033      // Disable readlogs and expect the unbind.
2034      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
2035      ASSERT_GE(mDataLoader->setStorageParams(false), 0);
2036  
2037      callback();
2038  
2039      // Destroyed.
2040      ASSERT_EQ(mDataLoader->status(), IDataLoaderStatusListener::DATA_LOADER_DESTROYED);
2041  }
2042  
createPerUidTimeouts(std::initializer_list<std::tuple<int,int,int,int>> tuples)2043  static std::vector<PerUidReadTimeouts> createPerUidTimeouts(
2044          std::initializer_list<std::tuple<int, int, int, int>> tuples) {
2045      std::vector<PerUidReadTimeouts> result;
2046      for (auto&& tuple : tuples) {
2047          result.emplace_back();
2048          auto& timeouts = result.back();
2049          timeouts.uid = std::get<0>(tuple);
2050          timeouts.minTimeUs = std::get<1>(tuple);
2051          timeouts.minPendingTimeUs = std::get<2>(tuple);
2052          timeouts.maxPendingTimeUs = std::get<3>(tuple);
2053      }
2054      return result;
2055  }
2056  
checkPerUidTimeouts(const Control & control,const std::vector<PerUidReadTimeouts> & perUidReadTimeouts)2057  static ErrorCode checkPerUidTimeouts(const Control& control,
2058                                       const std::vector<PerUidReadTimeouts>& perUidReadTimeouts) {
2059      std::vector<PerUidReadTimeouts> expected =
2060              createPerUidTimeouts({{0, 1, 2, 3}, {1, 2, 3, 4}, {2, 3, 4, 100000000}});
2061      EXPECT_EQ(expected, perUidReadTimeouts);
2062      return 0;
2063  }
2064  
checkPerUidTimeoutsEmpty(const Control & control,const std::vector<PerUidReadTimeouts> & perUidReadTimeouts)2065  static ErrorCode checkPerUidTimeoutsEmpty(
2066          const Control& control, const std::vector<PerUidReadTimeouts>& perUidReadTimeouts) {
2067      EXPECT_EQ(0u, perUidReadTimeouts.size());
2068      return 0;
2069  }
2070  
TEST_F(IncrementalServiceTest,testPerUidTimeoutsTooShort)2071  TEST_F(IncrementalServiceTest, testPerUidTimeoutsTooShort) {
2072      EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _, _)).Times(1);
2073      EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
2074      EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
2075      EXPECT_CALL(*mDataLoader, start(_)).Times(1);
2076      EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
2077      EXPECT_CALL(*mIncFs, setUidReadTimeouts(_, _)).Times(0);
2078      EXPECT_CALL(*mTimedQueue, addJob(_, _, _)).Times(2);
2079      EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
2080      TemporaryDir tempDir;
2081      int storageId =
2082              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
2083                                                 IncrementalService::CreateOptions::CreateNew);
2084      ASSERT_GE(storageId, 0);
2085      mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {}, {},
2086                                        createPerUidTimeouts(
2087                                                {{0, 1, 2, 3}, {1, 2, 3, 4}, {2, 3, 4, 5}}));
2088  }
2089  
TEST_F(IncrementalServiceTest,testPerUidTimeoutsSuccess)2090  TEST_F(IncrementalServiceTest, testPerUidTimeoutsSuccess) {
2091      mVold->setIncFsMountOptionsSuccess();
2092      mAppOpsManager->checkPermissionSuccess();
2093      mFs->hasFiles();
2094  
2095      EXPECT_CALL(*mIncFs, setUidReadTimeouts(_, _))
2096              // First call.
2097              .WillOnce(Invoke(&checkPerUidTimeouts))
2098              // Fully loaded and no readlogs.
2099              .WillOnce(Invoke(&checkPerUidTimeoutsEmpty));
2100      EXPECT_CALL(*mTimedQueue, addJob(_, _, _)).Times(3);
2101  
2102      // Loading storage.
2103      EXPECT_CALL(*mIncFs, isEverythingFullyLoaded(_))
2104              .WillOnce(Return(incfs::LoadingState::MissingBlocks))
2105              .WillOnce(Return(incfs::LoadingState::MissingBlocks))
2106              .WillOnce(Return(incfs::LoadingState::Full));
2107  
2108      // Mark DataLoader as 'system' so that readlogs don't pollute the timed queue.
2109      mDataLoaderParcel.packageName = "android";
2110  
2111      TemporaryDir tempDir;
2112      int storageId =
2113              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
2114                                                 IncrementalService::CreateOptions::CreateNew);
2115      ASSERT_GE(storageId, 0);
2116      mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {}, {},
2117                                        createPerUidTimeouts(
2118                                                {{0, 1, 2, 3}, {1, 2, 3, 4}, {2, 3, 4, 100000000}}));
2119  
2120      {
2121          // Timed callback present -> 0 progress.
2122          ASSERT_EQ(IncrementalService::kAllStoragesId, mTimedQueue->mId);
2123          ASSERT_GE(mTimedQueue->mAfter, std::chrono::seconds(1));
2124          const auto timedCallback = mTimedQueue->mWhat;
2125          mTimedQueue->clearJob(IncrementalService::kAllStoragesId);
2126  
2127          // Call it again.
2128          timedCallback();
2129      }
2130  
2131      {
2132          // Still present -> some progress.
2133          ASSERT_EQ(IncrementalService::kAllStoragesId, mTimedQueue->mId);
2134          ASSERT_GE(mTimedQueue->mAfter, std::chrono::seconds(1));
2135          const auto timedCallback = mTimedQueue->mWhat;
2136          mTimedQueue->clearJob(IncrementalService::kAllStoragesId);
2137  
2138          // Fully loaded but readlogs collection enabled.
2139          ASSERT_GE(mDataLoader->setStorageParams(true), 0);
2140  
2141          // Call it again.
2142          timedCallback();
2143      }
2144  
2145      {
2146          // Still present -> fully loaded + readlogs.
2147          ASSERT_EQ(IncrementalService::kAllStoragesId, mTimedQueue->mId);
2148          ASSERT_GE(mTimedQueue->mAfter, std::chrono::seconds(1));
2149          const auto timedCallback = mTimedQueue->mWhat;
2150          mTimedQueue->clearJob(IncrementalService::kAllStoragesId);
2151  
2152          // Now disable readlogs.
2153          ASSERT_GE(mDataLoader->setStorageParams(false), 0);
2154  
2155          // Call it again.
2156          timedCallback();
2157      }
2158  
2159      // No callbacks anymore -> fully loaded and no readlogs.
2160      ASSERT_EQ(mTimedQueue->mAfter, Milliseconds());
2161  }
2162  
TEST_F(IncrementalServiceTest,testInvalidMetricsQuery)2163  TEST_F(IncrementalServiceTest, testInvalidMetricsQuery) {
2164      const auto invalidStorageId = 100;
2165      android::os::PersistableBundle result{};
2166      mIncrementalService->getMetrics(invalidStorageId, &result);
2167      int64_t expected = -1, value = -1;
2168      ASSERT_FALSE(
2169              result.getLong(String16(BnIncrementalService::METRICS_MILLIS_SINCE_OLDEST_PENDING_READ()
2170                                              .c_str()),
2171                             &value));
2172      ASSERT_EQ(expected, value);
2173      ASSERT_TRUE(result.empty());
2174  }
2175  
TEST_F(IncrementalServiceTest,testNoDataLoaderMetrics)2176  TEST_F(IncrementalServiceTest, testNoDataLoaderMetrics) {
2177      mVold->setIncFsMountOptionsSuccess();
2178      TemporaryDir tempDir;
2179      int storageId =
2180              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
2181                                                 IncrementalService::CreateOptions::CreateNew);
2182      ASSERT_GE(storageId, 0);
2183      android::os::PersistableBundle result{};
2184      mIncrementalService->getMetrics(storageId, &result);
2185      int64_t expected = -1, value = -1;
2186      ASSERT_FALSE(
2187              result.getLong(String16(BnIncrementalService::METRICS_MILLIS_SINCE_OLDEST_PENDING_READ()
2188                                              .c_str()),
2189                             &value));
2190      ASSERT_EQ(expected, value);
2191      ASSERT_EQ(1, (int)result.size());
2192      bool expectedReadLogsEnabled = false;
2193      ASSERT_TRUE(
2194              result.getBoolean(String16(BnIncrementalService::METRICS_READ_LOGS_ENABLED().c_str()),
2195                                &expectedReadLogsEnabled));
2196      ASSERT_EQ(mVold->readLogsEnabled(), expectedReadLogsEnabled);
2197  }
2198  
TEST_F(IncrementalServiceTest,testInvalidMetricsKeys)2199  TEST_F(IncrementalServiceTest, testInvalidMetricsKeys) {
2200      mVold->setIncFsMountOptionsSuccess();
2201      TemporaryDir tempDir;
2202      int storageId =
2203              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
2204                                                 IncrementalService::CreateOptions::CreateNew);
2205      ASSERT_GE(storageId, 0);
2206      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
2207                                                    {}, {}));
2208      android::os::PersistableBundle result{};
2209      mIncrementalService->getMetrics(storageId, &result);
2210      int64_t expected = -1, value = -1;
2211      ASSERT_FALSE(result.getLong(String16("invalid"), &value));
2212      ASSERT_EQ(expected, value);
2213      ASSERT_EQ(6, (int)result.size());
2214  }
2215  
TEST_F(IncrementalServiceTest,testMetricsWithNoLastReadError)2216  TEST_F(IncrementalServiceTest, testMetricsWithNoLastReadError) {
2217      mVold->setIncFsMountOptionsSuccess();
2218      ON_CALL(*mIncFs, getMetrics(_))
2219              .WillByDefault(Return(Metrics{
2220                      .readsDelayedMin = 10,
2221                      .readsDelayedMinUs = 5000,
2222                      .readsDelayedPending = 10,
2223                      .readsDelayedPendingUs = 5000,
2224                      .readsFailedHashVerification = 10,
2225                      .readsFailedOther = 10,
2226                      .readsFailedTimedOut = 10,
2227              }));
2228      ON_CALL(*mIncFs, getLastReadError(_)).WillByDefault(Return(LastReadError{}));
2229      TemporaryDir tempDir;
2230      int storageId =
2231              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
2232                                                 IncrementalService::CreateOptions::CreateNew);
2233      ASSERT_GE(storageId, 0);
2234      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
2235                                                    {}, {}));
2236      android::os::PersistableBundle result{};
2237      mIncrementalService->getMetrics(storageId, &result);
2238      ASSERT_EQ(9, (int)result.size());
2239  
2240      int expectedtotalDelayedReads = 20, totalDelayedReads = -1;
2241      ASSERT_TRUE(result.getInt(String16(BnIncrementalService::METRICS_TOTAL_DELAYED_READS().c_str()),
2242                                &totalDelayedReads));
2243      ASSERT_EQ(expectedtotalDelayedReads, totalDelayedReads);
2244      int expectedtotalFailedReads = 30, totalFailedReads = -1;
2245      ASSERT_TRUE(result.getInt(String16(BnIncrementalService::METRICS_TOTAL_FAILED_READS().c_str()),
2246                                &totalFailedReads));
2247      ASSERT_EQ(expectedtotalFailedReads, totalFailedReads);
2248      int64_t expectedtotalDelayedReadsMillis = 10, totalDelayedReadsMillis = -1;
2249      ASSERT_TRUE(result.getLong(String16(BnIncrementalService::METRICS_TOTAL_DELAYED_READS_MILLIS()
2250                                                  .c_str()),
2251                                 &totalDelayedReadsMillis));
2252      ASSERT_EQ(expectedtotalDelayedReadsMillis, totalDelayedReadsMillis);
2253  
2254      int64_t expectedMillisSinceLastReadError = -1, millisSinceLastReadError = -1;
2255      ASSERT_FALSE(
2256              result.getLong(String16(BnIncrementalService::METRICS_MILLIS_SINCE_LAST_READ_ERROR()
2257                                              .c_str()),
2258                             &millisSinceLastReadError));
2259      ASSERT_EQ(expectedMillisSinceLastReadError, millisSinceLastReadError);
2260      int expectedLastReadErrorNumber = -1, lastReadErrorNumber = -1;
2261      ASSERT_FALSE(
2262              result.getInt(String16(BnIncrementalService::METRICS_LAST_READ_ERROR_NUMBER().c_str()),
2263                            &lastReadErrorNumber));
2264      ASSERT_EQ(expectedLastReadErrorNumber, lastReadErrorNumber);
2265      int expectedLastReadUid = -1, lastReadErrorUid = -1;
2266      ASSERT_FALSE(
2267              result.getInt(String16(BnIncrementalService::METRICS_LAST_READ_ERROR_UID().c_str()),
2268                            &lastReadErrorUid));
2269      ASSERT_EQ(expectedLastReadUid, lastReadErrorUid);
2270  }
2271  
TEST_F(IncrementalServiceTest,testMetricsWithLastReadError)2272  TEST_F(IncrementalServiceTest, testMetricsWithLastReadError) {
2273      mVold->setIncFsMountOptionsSuccess();
2274      ON_CALL(*mIncFs, getMetrics(_)).WillByDefault(Return(Metrics{}));
2275      mClock->advanceMs(5);
2276      const auto now = mClock->getClock();
2277      ON_CALL(*mIncFs, getLastReadError(_))
2278              .WillByDefault(Return(LastReadError{.timestampUs = static_cast<uint64_t>(
2279                                                          duration_cast<std::chrono::microseconds>(
2280                                                                  now.time_since_epoch())
2281                                                                  .count()),
2282                                                  .errorNo = static_cast<uint32_t>(-ETIME),
2283                                                  .uid = 20000}));
2284      TemporaryDir tempDir;
2285      int storageId =
2286              mIncrementalService->createStorage(tempDir.path, mDataLoaderParcel,
2287                                                 IncrementalService::CreateOptions::CreateNew);
2288      ASSERT_GE(storageId, 0);
2289      ASSERT_TRUE(mIncrementalService->startLoading(storageId, std::move(mDataLoaderParcel), {}, {},
2290                                                    {}, {}));
2291      mClock->advanceMs(10);
2292      android::os::PersistableBundle result{};
2293      mIncrementalService->getMetrics(storageId, &result);
2294      ASSERT_EQ(12, (int)result.size());
2295      int64_t expectedMillisSinceLastReadError = 10, millisSinceLastReadError = -1;
2296      ASSERT_TRUE(result.getLong(String16(BnIncrementalService::METRICS_MILLIS_SINCE_LAST_READ_ERROR()
2297                                                  .c_str()),
2298                                 &millisSinceLastReadError));
2299      ASSERT_EQ(expectedMillisSinceLastReadError, millisSinceLastReadError);
2300      int expectedLastReadErrorNumber = -ETIME, lastReadErrorNumber = -1;
2301      ASSERT_TRUE(
2302              result.getInt(String16(BnIncrementalService::METRICS_LAST_READ_ERROR_NUMBER().c_str()),
2303                            &lastReadErrorNumber));
2304      ASSERT_EQ(expectedLastReadErrorNumber, lastReadErrorNumber);
2305      int expectedLastReadUid = 20000, lastReadErrorUid = -1;
2306      ASSERT_TRUE(result.getInt(String16(BnIncrementalService::METRICS_LAST_READ_ERROR_UID().c_str()),
2307                                &lastReadErrorUid));
2308      ASSERT_EQ(expectedLastReadUid, lastReadErrorUid);
2309  }
2310  
2311  } // namespace android::os::incremental
2312