View Javadoc
1   /*
2    * Copyright (C) 2010, Robin Rosenberg and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  package org.eclipse.jgit.internal.storage.file;
11  
12  import static org.eclipse.jgit.junit.JGitTestUtil.read;
13  import static org.eclipse.jgit.junit.JGitTestUtil.write;
14  import static org.junit.Assert.assertEquals;
15  import static org.junit.Assert.assertFalse;
16  import static org.junit.Assert.assertTrue;
17  import static org.junit.Assert.fail;
18  
19  import java.io.File;
20  import java.io.IOException;
21  import java.io.OutputStream;
22  import java.nio.file.Files;
23  import java.nio.file.Path;
24  import java.nio.file.StandardCopyOption;
25  import java.nio.file.StandardOpenOption;
26  import java.nio.file.attribute.FileTime;
27  import java.time.Duration;
28  import java.time.Instant;
29  import java.util.ArrayList;
30  import java.util.concurrent.TimeUnit;
31  
32  import org.eclipse.jgit.junit.MockSystemReader;
33  import org.eclipse.jgit.util.FS;
34  import org.eclipse.jgit.util.FS.FileStoreAttributes;
35  import org.eclipse.jgit.util.FileUtils;
36  import org.eclipse.jgit.util.Stats;
37  import org.eclipse.jgit.util.SystemReader;
38  import org.junit.After;
39  import org.junit.Assume;
40  import org.junit.Before;
41  import org.junit.Test;
42  import org.slf4j.Logger;
43  import org.slf4j.LoggerFactory;
44  
45  public class FileSnapshotTest {
46  	private static final Logger LOG = LoggerFactory
47  			.getLogger(FileSnapshotTest.class);
48  
49  	private Path trash;
50  
51  	private FileStoreAttributes fsAttrCache;
52  
53  	@Before
54  	public void setUp() throws Exception {
55  		SystemReader.setInstance(new MockSystemReader());
56  		trash = Files.createTempDirectory("tmp_");
57  		// measure timer resolution before the test to avoid time critical tests
58  		// are affected by time needed for measurement
59  		fsAttrCache = FS
60  				.getFileStoreAttributes(trash.getParent());
61  	}
62  
63  	@Before
64  	@After
65  	public void tearDown() throws Exception {
66  		FileUtils.delete(trash.toFile(),
67  				FileUtils.RECURSIVE | FileUtils.SKIP_MISSING);
68  	}
69  
70  	private static void waitNextTick(Path f) throws IOException {
71  		Instant initialLastModified = FS.DETECTED.lastModifiedInstant(f);
72  		do {
73  			FS.DETECTED.setLastModified(f, Instant.now());
74  		} while (FS.DETECTED.lastModifiedInstant(f)
75  				.equals(initialLastModified));
76  	}
77  
78  	/**
79  	 * Change data and time stamp.
80  	 *
81  	 * @throws Exception
82  	 */
83  	@Test
84  	public void testActuallyIsModifiedTrivial() throws Exception {
85  		Path f1 = createFile("simple");
86  		waitNextTick(f1);
87  		FileSnapshot save = FileSnapshot.save(f1.toFile());
88  		append(f1, (byte) 'x');
89  		waitNextTick(f1);
90  		assertTrue(save.isModified(f1.toFile()));
91  	}
92  
93  	/**
94  	 * Create a file, but don't wait long enough for the difference between file
95  	 * system clock and system clock to be significant. Assume the file may have
96  	 * been modified. It may have been, but the clock alone cannot determine
97  	 * this
98  	 *
99  	 * @throws Exception
100 	 */
101 	@Test
102 	public void testNewFileWithWait() throws Exception {
103 		// if filesystem timestamp resolution is high the snapshot won't be
104 		// racily clean
105 		Assume.assumeTrue(
106 				fsAttrCache.getFsTimestampResolution()
107 						.compareTo(Duration.ofMillis(10)) > 0);
108 		Path f1 = createFile("newfile");
109 		waitNextTick(f1);
110 		FileSnapshot save = FileSnapshot.save(f1.toFile());
111 		TimeUnit.NANOSECONDS.sleep(
112 				fsAttrCache.getFsTimestampResolution().dividedBy(2).toNanos());
113 		assertTrue(save.isModified(f1.toFile()));
114 	}
115 
116 	/**
117 	 * Same as {@link #testNewFileWithWait()} but do not wait at all
118 	 *
119 	 * @throws Exception
120 	 */
121 	@Test
122 	public void testNewFileNoWait() throws Exception {
123 		// if filesystem timestamp resolution is smaller than time needed to
124 		// create a file and FileSnapshot the snapshot won't be racily clean
125 		Assume.assumeTrue(fsAttrCache.getFsTimestampResolution()
126 				.compareTo(Duration.ofMillis(10)) > 0);
127 		for (int i = 0; i < 50; i++) {
128 			Instant start = Instant.now();
129 			Path f1 = createFile("newfile");
130 			FileSnapshot save = FileSnapshot.save(f1.toFile());
131 			Duration res = FS.getFileStoreAttributes(f1)
132 					.getFsTimestampResolution();
133 			Instant end = Instant.now();
134 			if (Duration.between(start, end)
135 					.compareTo(res.multipliedBy(2)) > 0) {
136 				// This test is racy: under load, there may be a delay between createFile() and
137 				// FileSnapshot.save(). This can stretch the time between the read TS and FS
138 				// creation TS to the point that it exceeds the FS granularity, and we
139 				// conclude it cannot be racily clean, and therefore must be really clean.
140 				//
141 				// This should be relatively uncommon.
142 				continue;
143 			}
144 			// The file wasn't really modified, but it looks just like a "maybe racily clean"
145 			// file.
146 			assertTrue(save.isModified(f1.toFile()));
147 			return;
148 		}
149 		fail("too much load for this test");
150 	}
151 
152 	/**
153 	 * Simulate packfile replacement in same file which may occur if set of
154 	 * objects in the pack is the same but pack config was different. On Posix
155 	 * filesystems this should change the inode (filekey in java.nio
156 	 * terminology).
157 	 *
158 	 * @throws Exception
159 	 */
160 	@Test
161 	public void testSimulatePackfileReplacement() throws Exception {
162 		Assume.assumeFalse(SystemReader.getInstance().isWindows());
163 		Path f1 = createFile("file"); // inode y
164 		Path f2 = createFile("fool"); // Guarantees new inode x
165 		// wait on f2 since this method resets lastModified of the file
166 		// and leaves lastModified of f1 untouched
167 		waitNextTick(f2);
168 		waitNextTick(f2);
169 		FileTime timestamp = Files.getLastModifiedTime(f1);
170 		FileSnapshot save = FileSnapshot.save(f1.toFile());
171 		Files.move(f2, f1, // Now "file" is inode x
172 				StandardCopyOption.REPLACE_EXISTING,
173 				StandardCopyOption.ATOMIC_MOVE);
174 		Files.setLastModifiedTime(f1, timestamp);
175 		assertTrue(save.isModified(f1.toFile()));
176 		assertTrue("unexpected change of fileKey", save.wasFileKeyChanged());
177 		assertFalse("unexpected size change", save.wasSizeChanged());
178 		assertFalse("unexpected lastModified change",
179 				save.wasLastModifiedChanged());
180 		assertFalse("lastModified was unexpectedly racily clean",
181 				save.wasLastModifiedRacilyClean());
182 	}
183 
184 	/**
185 	 * Append a character to a file to change its size and set original
186 	 * lastModified
187 	 *
188 	 * @throws Exception
189 	 */
190 	@Test
191 	public void testFileSizeChanged() throws Exception {
192 		Path f = createFile("file");
193 		FileTime timestamp = Files.getLastModifiedTime(f);
194 		FileSnapshot save = FileSnapshot.save(f.toFile());
195 		append(f, (byte) 'x');
196 		Files.setLastModifiedTime(f, timestamp);
197 		assertTrue(save.isModified(f.toFile()));
198 		assertTrue(save.wasSizeChanged());
199 	}
200 
201 	@Test
202 	public void fileSnapshotEquals() throws Exception {
203 		// 0 sized FileSnapshot.
204 		FileSnapshot fs1 = FileSnapshot.MISSING_FILE;
205 		// UNKNOWN_SIZE FileSnapshot.
206 		FileSnapshot fs2 = FileSnapshot.save(fs1.lastModifiedInstant());
207 
208 		assertTrue(fs1.equals(fs2));
209 		assertTrue(fs2.equals(fs1));
210 	}
211 
212 	@SuppressWarnings("boxing")
213 	@Test
214 	public void detectFileModified() throws IOException {
215 		int failures = 0;
216 		long racyNanos = 0;
217 		final int COUNT = 10000;
218 		ArrayList<Long> deltas = new ArrayList<>();
219 		File f = createFile("test").toFile();
220 		for (int i = 0; i < COUNT; i++) {
221 			write(f, "a");
222 			FileSnapshot snapshot = FileSnapshot.save(f);
223 			assertEquals("file should contain 'a'", "a", read(f));
224 			write(f, "b");
225 			if (!snapshot.isModified(f)) {
226 				deltas.add(snapshot.lastDelta());
227 				racyNanos = snapshot.lastRacyThreshold();
228 				failures++;
229 			}
230 			assertEquals("file should contain 'b'", "b", read(f));
231 		}
232 		if (failures > 0) {
233 			Stats stats = new Stats();
234 			LOG.debug(
235 					"delta [ns] since modification FileSnapshot failed to detect");
236 			for (Long d : deltas) {
237 				stats.add(d);
238 				LOG.debug(String.format("%,d", d));
239 			}
240 			LOG.error(
241 					"count, failures, eff. racy threshold [ns], delta min [ns],"
242 							+ " delta max [ns], delta avg [ns],"
243 							+ " delta stddev [ns]");
244 			LOG.error(String.format(
245 					"%,d, %,d, %,d, %,.0f, %,.0f, %,.0f, %,.0f", COUNT,
246 					failures, racyNanos, stats.min(), stats.max(),
247 					stats.avg(), stats.stddev()));
248 		}
249 		assertTrue(
250 				String.format(
251 						"FileSnapshot: failures to detect file modifications"
252 								+ " %d out of %d\n"
253 								+ "timestamp resolution %d µs"
254 								+ " min racy threshold %d µs"
255 						, failures, COUNT,
256 						fsAttrCache.getFsTimestampResolution().toNanos() / 1000,
257 						fsAttrCache.getMinimalRacyInterval().toNanos() / 1000),
258 				failures == 0);
259 	}
260 
261 	private Path createFile(String string) throws IOException {
262 		Files.createDirectories(trash);
263 		return Files.createTempFile(trash, string, "tdat");
264 	}
265 
266 	private static void append(Path f, byte b) throws IOException {
267 		try (OutputStream os = Files.newOutputStream(f,
268 				StandardOpenOption.APPEND)) {
269 			os.write(b);
270 		}
271 	}
272 
273 }