Handle resources, never left left opened file handler on windows (#1857)

On windows unit tests are failing because junit checks if all @tempdir directries are empty and can be deleted after test run.
But due to opened file handles (not closed resource streams) Windows keeps files, which are "in use".
Linux is less strict in this area.
Additionally I want highlight that XMLStreamReaderImpl/XMLStreamWriterImpl from apache.xerces library (in OpenJDK11 at least) which are picked at runtime as xml parser implementation - they don't close associated resources.
BTW, I thing that relying on some runtime (sometimes - unpredictable) dependencies - is bad practice, but this it up to separate topic.
Additional fix: in IOUtil is file is locked (due to permissions or opened handle) - it will undlessly try-and-retry to delete it until end of the world, on windows.
This commit is contained in:
StNekroman
2021-11-22 11:22:46 +02:00
committed by GitHub
parent 47842cc071
commit b09284f1f5
12 changed files with 150 additions and 147 deletions

View File

@@ -391,9 +391,7 @@ public final class IOUtil
}
}
for (int i = 20; !file.delete(); i--)
{
if (i <= 20)
for (int i = 20; !file.delete() && i > 0; i--)
{
String message = "could not delete file ".concat(file.getPath());
@@ -405,7 +403,6 @@ public final class IOUtil
{
throw new IOException(message);
}
}
try
{

View File

@@ -28,6 +28,8 @@ import com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.nio.file.Paths;
import java.util.Map;
import java.util.Properties;
@@ -38,19 +40,19 @@ class BaseDirectoryTest {
@Test
void shouldGetFromClassPathResource() {
BaseDirectory directory = builder().withClassPathResource("/sonia/scm/basedirectory.properties").create();
assertThat(directory.find().toString()).isEqualTo("/tmp/scm_home");
assertThat(directory.find()).isEqualTo(Paths.get("/tmp/scm_home"));
}
@Test
void shouldGetFromSystemProperty() {
BaseDirectory directory = builder().withSystemProperty(BaseDirectory.SYSTEM_PROPERTY, "/tmp/scm_home").create();
assertThat(directory.find().toString()).isEqualTo("/tmp/scm_home");
assertThat(directory.find()).isEqualTo(Paths.get("/tmp/scm_home"));
}
@Test
void shouldGetFromEnvironmentVariable() {
BaseDirectory directory = builder().withEnvironment(BaseDirectory.ENVIRONMENT_VARIABLE, "/tmp/scm_home").create();
assertThat(directory.find().toString()).isEqualTo("/tmp/scm_home");
assertThat(directory.find()).isEqualTo(Paths.get("/tmp/scm_home"));
}
@Nested
@@ -59,19 +61,19 @@ class BaseDirectoryTest {
@Test
void linux() {
BaseDirectory directory = builder().withSystemProperty("user.home", "/tmp").create();
assertThat(directory.find().toString()).isEqualTo("/tmp/.scm");
assertThat(directory.find()).isEqualTo(Paths.get("/tmp/.scm"));
}
@Test
void osx() {
BaseDirectory directory = builder().withOsx().withSystemProperty("user.home", "/tmp").create();
assertThat(directory.find().toString()).isEqualTo("/tmp/Library/Application Support/SCM-Manager");
assertThat(directory.find()).isEqualTo(Paths.get("/tmp/Library/Application Support/SCM-Manager"));
}
@Test
void windows() {
BaseDirectory directory = builder().withWindows().withEnvironment("APPDATA", "/tmp").create();
assertThat(directory.find().toString()).isEqualTo("/tmp\\SCM-Manager");
assertThat(directory.find()).isEqualTo(Paths.get("/tmp\\SCM-Manager"));
}
}

View File

@@ -43,6 +43,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Path;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@@ -137,17 +138,14 @@ class SmpArchiveTest {
return archiveFile;
}
private XMLStreamWriter createStreamWriter(File file) throws IOException, XMLStreamException {
return XMLOutputFactory.newFactory().createXMLStreamWriter(new FileOutputStream(file));
}
private void writeDescriptor(File descriptor, String name, String version) throws IOException, XMLStreamException {
IOUtil.mkdirs(descriptor.getParentFile());
XMLStreamWriter writer = null;
try {
writer = createStreamWriter(descriptor);
try (OutputStream os = new FileOutputStream(descriptor)) {
writer = XMLOutputFactory.newFactory().createXMLStreamWriter(os);
writer.writeStartDocument();
writer.writeStartElement("plugin");
writer.writeStartElement("information");
@@ -157,12 +155,9 @@ class SmpArchiveTest {
writer.writeEndElement();
writer.writeEndElement();
writer.writeEndDocument();
} finally {
if (writer != null) {
writer.close();
}
}
}
private void writeElement(XMLStreamWriter writer, String name, String value) throws XMLStreamException {
if (!Strings.isNullOrEmpty(value)) {

View File

@@ -36,6 +36,9 @@ import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.stream.XMLStreamWriter;
import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -67,7 +70,8 @@ class PathDatabase {
CopyOnWrite.withTemporaryFile(
temp -> {
try (IndentXMLStreamWriter writer = XmlStreams.createWriter(temp)) {
try (Writer ioWriter = Files.newBufferedWriter(temp, StandardCharsets.UTF_8)) {
try (IndentXMLStreamWriter writer = XmlStreams.createWriter(ioWriter)) {
writer.writeStartDocument(ENCODING, VERSION);
writeRepositoriesStart(writer, creationTime, lastModified);
@@ -77,13 +81,14 @@ class PathDatabase {
writer.writeEndElement();
writer.writeEndDocument();
} catch (XMLStreamException | IOException ex) {
} catch (XMLStreamException ex) {
throw new InternalRepositoryException(
ContextEntry.ContextBuilder.entity(Path.class, storePath.toString()).build(),
"failed to write repository path database",
ex
);
}
}
},
storePath
);
@@ -121,8 +126,8 @@ class PathDatabase {
void read(OnRepositories onRepositories, OnRepository onRepository) {
LOG.trace("read repository path database from {}", storePath);
XMLStreamReader reader = null;
try {
reader = XmlStreams.createReader(storePath);
try (Reader inputReader = Files.newBufferedReader(storePath, StandardCharsets.UTF_8)) {
reader = XmlStreams.createReader(inputReader);
while (reader.hasNext()) {
int eventType = reader.next();

View File

@@ -35,6 +35,8 @@ import javax.inject.Inject;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -135,8 +137,8 @@ public class FileStoreExporter implements StoreExporter {
private StoreType determineConfigType(Path storePath) {
XMLStreamReader reader = null;
try {
reader = XmlStreams.createReader(storePath);
try (Reader inputReader = Files.newBufferedReader(storePath, StandardCharsets.UTF_8)) {
reader = XmlStreams.createReader(inputReader);
reader.nextTag();
if (
"configuration".equals(reader.getLocalName())

View File

@@ -37,6 +37,10 @@ import javax.xml.bind.Marshaller;
import javax.xml.namespace.QName;
import javax.xml.stream.XMLStreamReader;
import java.io.File;
import java.io.Reader;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
@@ -138,8 +142,9 @@ public class JAXBConfigurationEntryStore<V> implements ConfigurationEntryStore<V
context.withUnmarshaller(u -> {
XMLStreamReader reader = null;
try {
reader = XmlStreams.createReader(file);
try (Reader inputReader = Files.newBufferedReader(file.toPath(), StandardCharsets.UTF_8)) {
reader = XmlStreams.createReader(inputReader);
// configuration
reader.nextTag();
@@ -190,7 +195,8 @@ public class JAXBConfigurationEntryStore<V> implements ConfigurationEntryStore<V
CopyOnWrite.withTemporaryFile(
temp -> {
try (IndentXMLStreamWriter writer = XmlStreams.createWriter(temp)) {
try (Writer ioWriter = Files.newBufferedWriter(temp, StandardCharsets.UTF_8);
IndentXMLStreamWriter writer = XmlStreams.createWriter(ioWriter)) {
writer.writeStartDocument();
// configuration start

View File

@@ -32,13 +32,8 @@ import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.stream.XMLStreamWriter;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
public final class XmlStreams {
@@ -67,31 +62,13 @@ public final class XmlStreams {
}
}
public static XMLStreamReader createReader(Path path) throws IOException, XMLStreamException {
return createReader(Files.newBufferedReader(path, StandardCharsets.UTF_8));
}
public static XMLStreamReader createReader(File file) throws IOException, XMLStreamException {
return createReader(file.toPath());
}
private static XMLStreamReader createReader(Reader reader) throws XMLStreamException {
public static XMLStreamReader createReader(Reader reader) throws XMLStreamException {
XMLInputFactory factory = XMLInputFactory.newInstance();
factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE);
return factory.createXMLStreamReader(reader);
}
public static IndentXMLStreamWriter createWriter(Path path) throws IOException, XMLStreamException {
return createWriter(Files.newBufferedWriter(path, StandardCharsets.UTF_8));
}
public static IndentXMLStreamWriter createWriter(File file) throws IOException, XMLStreamException {
return createWriter(file.toPath());
}
private static IndentXMLStreamWriter createWriter(Writer writer) throws XMLStreamException {
public static IndentXMLStreamWriter createWriter(Writer writer) throws XMLStreamException {
return new IndentXMLStreamWriter(XMLOutputFactory.newFactory().createXMLStreamWriter(writer));
}
}

View File

@@ -30,6 +30,7 @@ import org.junit.jupiter.api.io.TempDir;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -43,9 +44,11 @@ class CopyOnWriteTest {
void shouldCreateNewFile(@TempDir Path tempDir) {
Path expectedFile = tempDir.resolve("toBeCreated.txt");
withTemporaryFile(
file -> new FileOutputStream(file.toFile()).write("great success".getBytes()),
expectedFile);
withTemporaryFile(file -> {
try (OutputStream os = new FileOutputStream(file.toFile())) {
os.write("great success".getBytes());
}
}, expectedFile);
Assertions.assertThat(expectedFile).hasContent("great success");
}
@@ -55,34 +58,40 @@ class CopyOnWriteTest {
Path expectedFile = tempDir.resolve("toBeOverwritten.txt");
Files.createFile(expectedFile);
withTemporaryFile(
file -> new FileOutputStream(file.toFile()).write("great success".getBytes()),
expectedFile);
withTemporaryFile(file -> {
try (OutputStream os = new FileOutputStream(file.toFile())) {
os.write("great success".getBytes());
}
}, expectedFile);
Assertions.assertThat(expectedFile).hasContent("great success");
}
@Test
void shouldFailForDirectory(@TempDir Path tempDir) {
assertThrows(IllegalArgumentException.class,
() -> withTemporaryFile(
file -> new FileOutputStream(file.toFile()).write("should not be written".getBytes()),
tempDir));
assertThrows(IllegalArgumentException.class, () -> withTemporaryFile(file -> {
try (OutputStream os = new FileOutputStream(file.toFile())) {
os.write("should not be written".getBytes());
}
}, tempDir));
}
@Test
void shouldFailForMissingDirectory() {
assertThrows(
IllegalArgumentException.class,
() -> withTemporaryFile(
file -> new FileOutputStream(file.toFile()).write("should not be written".getBytes()),
Paths.get("someFile")));
assertThrows(IllegalArgumentException.class, () -> withTemporaryFile(file -> {
try (OutputStream os = new FileOutputStream(file.toFile())) {
os.write("should not be written".getBytes());
}
}, Paths.get("someFile")));
}
@Test
void shouldKeepBackupIfTemporaryFileCouldNotBeWritten(@TempDir Path tempDir) throws IOException {
Path unchangedOriginalFile = tempDir.resolve("notToBeDeleted.txt");
new FileOutputStream(unchangedOriginalFile.toFile()).write("this should be kept".getBytes());
try (OutputStream unchangedOriginalOs = new FileOutputStream(unchangedOriginalFile.toFile())) {
unchangedOriginalOs.write("this should be kept".getBytes());
}
assertThrows(
StoreException.class,
@@ -111,7 +120,10 @@ class CopyOnWriteTest {
@Test
void shouldKeepBackupIfTemporaryFileIsMissing(@TempDir Path tempDir) throws IOException {
Path backedUpFile = tempDir.resolve("notToBeDeleted.txt");
new FileOutputStream(backedUpFile.toFile()).write("this should be kept".getBytes());
try (OutputStream backedUpOs = new FileOutputStream(backedUpFile.toFile())) {
backedUpOs.write("this should be kept".getBytes());
}
assertThrows(
StoreException.class,
@@ -125,11 +137,16 @@ class CopyOnWriteTest {
@Test
void shouldDeleteExistingFile(@TempDir Path tempDir) throws IOException {
Path expectedFile = tempDir.resolve("toBeReplaced.txt");
new FileOutputStream(expectedFile.toFile()).write("this should be removed".getBytes());
withTemporaryFile(
file -> new FileOutputStream(file.toFile()).write("overwritten".getBytes()),
expectedFile);
try (OutputStream expectedOs = new FileOutputStream(expectedFile.toFile())) {
expectedOs.write("this should be removed".getBytes());
}
withTemporaryFile(file -> {
try (OutputStream os = new FileOutputStream(file.toFile())) {
os.write("overwritten".getBytes()) ;
}
}, expectedFile);
Assertions.assertThat(Files.list(tempDir)).hasSize(1);
}

View File

@@ -93,8 +93,8 @@ class RepositoryImportStep implements ImportStep {
private void importFromTemporaryPath(ImportState state, Path path) {
LOG.debug("Importing repository from temporary location in work dir");
state.getLogger().step("importing repository from temporary location");
try {
unbundleRepository(state, Files.newInputStream(path));
try (InputStream is = Files.newInputStream(path)) {
unbundleRepository(state, is);
} catch (IOException e) {
throw new ImportFailedException(
entity(state.getRepository()).build(),

View File

@@ -34,17 +34,19 @@ import sonia.scm.version.Version;
import sonia.scm.xml.XmlStreams;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.stream.Stream;
@@ -74,8 +76,8 @@ abstract class DifferentiateBetweenConfigAndConfigEntryUpdateStep {
private boolean isConfigEntryFile(Path potentialFile) {
LOG.trace("Testing whether file is config entry file without mark: {}", potentialFile);
XMLStreamReader reader = null;
try {
reader = XmlStreams.createReader(potentialFile);
try (Reader inputReader = Files.newBufferedReader(potentialFile, StandardCharsets.UTF_8)) {
reader = XmlStreams.createReader(inputReader);
reader.nextTag();
if ("configuration".equals(reader.getLocalName())) {
reader.nextTag();
@@ -104,26 +106,25 @@ abstract class DifferentiateBetweenConfigAndConfigEntryUpdateStep {
}
private void writeXmlDocument(Document configEntryDocument, Path temporaryFile) throws TransformerException {
try {
TransformerFactory factory = TransformerFactory.newInstance();
try (OutputStream os = Files.newOutputStream(temporaryFile)) {
final TransformerFactory factory = TransformerFactory.newInstance();
factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
Transformer transformer = factory.newTransformer();
DOMSource domSource = new DOMSource(configEntryDocument);
StreamResult streamResult = new StreamResult(Files.newOutputStream(temporaryFile));
transformer.transform(domSource, streamResult);
final DOMSource domSource = new DOMSource(configEntryDocument);
factory.newTransformer().transform(domSource, new StreamResult(os));
} catch (IOException e) {
throw new UpdateException("Could not write modified config entry file", e);
}
}
private Document readAsXmlDocument(Path configFile) {
try {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
DocumentBuilder builder = factory.newDocumentBuilder();
return builder.parse(Files.newInputStream(configFile));
try (InputStream is = Files.newInputStream(configFile)) {
return factory.newDocumentBuilder().parse(is);
} catch (ParserConfigurationException | SAXException | IOException e) {
throw new UpdateException("Could not read config entry file", e);
}

View File

@@ -91,14 +91,8 @@ class CopyMigrationStrategyTest {
}
private void assertDirectoriesEqual(Path targetDataDir, Path originalDataDir) {
Stream<Path> list = null;
try {
list = Files.list(originalDataDir);
} catch (IOException e) {
fail("could not read original directory", e);
}
list.forEach(
original -> {
try (Stream<Path> list = Files.list(originalDataDir)) {
list.forEach(original -> {
Path expectedTarget = targetDataDir.resolve(original.getFileName());
assertThat(expectedTarget).exists();
if (Files.isDirectory(original)) {
@@ -106,7 +100,9 @@ class CopyMigrationStrategyTest {
} else {
assertThat(expectedTarget).hasSameContentAs(original);
}
});
} catch (IOException e) {
fail("could not read original directory", e);
}
);
}
}

View File

@@ -29,6 +29,8 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -43,16 +45,17 @@ import static org.assertj.core.api.Assertions.assertThat;
class DifferentiateBetweenConfigAndConfigEntryUpdateStepTest {
@Test
void shouldNotModifyConfigFile(@TempDir Path temp) throws IOException {
void shouldNotModifyConfigFile(@TempDir Path temp) throws IOException, URISyntaxException {
Path configFile = temp.resolve("some.store.xml");
copy(
getResource("sonia/scm/update/store/config_file.xml.content"),
newOutputStream(configFile));
try (OutputStream os = newOutputStream(configFile)) {
copy(getResource("sonia/scm/update/store/config_file.xml.content"), os);
new DifferentiateBetweenConfigAndConfigEntryUpdateStep() {}.updateAllInDirectory(temp);
assertContent(configFile, "sonia/scm/update/store/config_file.xml.content");
}
}
@Test
void shouldNotModifySingleLineFile(@TempDir Path temp) throws IOException {
@@ -77,33 +80,35 @@ class DifferentiateBetweenConfigAndConfigEntryUpdateStepTest {
}
@Test
void shouldIgnoreFilesWithoutXmlSuffix(@TempDir Path temp) throws IOException {
void shouldIgnoreFilesWithoutXmlSuffix(@TempDir Path temp) throws IOException, URISyntaxException {
Path otherFile = temp.resolve("some.other.file");
copy(
getResource("sonia/scm/update/store/config_entry_file_without_mark.xml.content"),
newOutputStream(otherFile));
try (OutputStream os = newOutputStream(otherFile)) {
copy(getResource("sonia/scm/update/store/config_entry_file_without_mark.xml.content"), os);
new DifferentiateBetweenConfigAndConfigEntryUpdateStep() {}.updateAllInDirectory(temp);
assertContent(otherFile, "sonia/scm/update/store/config_entry_file_without_mark.xml.content");
}
}
@Test
void shouldHandleConfigEntryFile(@TempDir Path temp) throws IOException {
void shouldHandleConfigEntryFile(@TempDir Path temp) throws IOException, URISyntaxException {
Path configFile = temp.resolve("some.store.xml");
copy(
getResource("sonia/scm/update/store/config_entry_file_without_mark.xml.content"),
newOutputStream(configFile));
try (OutputStream os = newOutputStream(configFile)) {
copy(getResource("sonia/scm/update/store/config_entry_file_without_mark.xml.content"), os);
new DifferentiateBetweenConfigAndConfigEntryUpdateStep() {}.updateAllInDirectory(temp);
assertThat(Files.readAllLines(configFile))
.areAtLeastOne(new Condition<>(line -> line.contains("<configuration type=\"config-entry\">"), "line containing start tag with attribute"))
.endsWith(Files.readAllLines(Paths.get(getResource("sonia/scm/update/store/config_entry_file.xml.content").getFile())).toArray(new String[9]));
.endsWith(Files.readAllLines(Paths.get(getResource("sonia/scm/update/store/config_entry_file.xml.content").toURI())).toArray(new String[9]));
}
}
private void assertContent(Path configFile, String expectedContentResource) {
private void assertContent(Path configFile, String expectedContentResource) throws URISyntaxException {
assertThat(configFile)
.hasSameTextualContentAs(Paths.get(getResource(expectedContentResource).getFile()));
.hasSameTextualContentAs(Paths.get(getResource(expectedContentResource).toURI()));
}
}