Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
a370a7b
WIP reconfigure only changed namespaces
attilakreiner May 31, 2024
3757300
WIP watch resources
attilakreiner Jun 5, 2024
c02e5c4
WIP refactoring watch resources
attilakreiner Jun 5, 2024
8fcae70
WIP refactoring 2
attilakreiner Jun 5, 2024
96f070e
WIP fix
attilakreiner Jun 6, 2024
df0b998
WIP EngineConfigWatcher
attilakreiner Jun 6, 2024
c0a554f
WIP ResourceResolver
attilakreiner Jun 6, 2024
b726267
WIP fix ResourceResolver
attilakreiner Jun 6, 2024
9c2cbbe
fix indentation
attilakreiner Jun 10, 2024
abb923f
fix EngineManager method names
attilakreiner Jun 10, 2024
b9355c6
rm ResourceResolver
attilakreiner Jun 10, 2024
c58ebc9
WIP EngineManager addResources
attilakreiner Jun 10, 2024
2285123
Revert "WIP reconfigure only changed namespaces"
attilakreiner Jun 11, 2024
39f34d0
fix EngineConfigWatcher addResources
attilakreiner Jun 11, 2024
b482fac
Merge branch 'develop' into config-reload
attilakreiner Jun 11, 2024
fd17488
Merge branch 'develop' into config-reload
attilakreiner Jun 13, 2024
b6c2ff2
Merge branch 'develop' into config-reload
attilakreiner Jun 14, 2024
2da3603
WIP readPath resolvePath readLocation
attilakreiner Jun 11, 2024
c85bdc1
WIP filesystem-http
attilakreiner Jun 5, 2024
ab94dac
WIP
attilakreiner Jun 18, 2024
22a2d03
fix
attilakreiner Jun 18, 2024
dfe9610
Merge branch 'develop' into config-reload
attilakreiner Jun 18, 2024
e4cf407
fix
attilakreiner Jun 18, 2024
df172d8
fix EngineConfiguration configPath
attilakreiner Jun 19, 2024
bef04ad
WIP fix
attilakreiner Jun 19, 2024
3177c80
WIP hfs 1
attilakreiner Jun 20, 2024
2a217f8
WIP hfs uri
attilakreiner Jun 20, 2024
4f5737a
WIP hfs resolveSibling
attilakreiner Jun 20, 2024
6395780
WIP hfs rm timeout
attilakreiner Jun 20, 2024
54a2b87
WIP hfs revert send
attilakreiner Jun 20, 2024
a5d0a39
WIP hfs HP readBody resolveBody
attilakreiner Jun 20, 2024
f19e623
WIP hfs HP readBody etag
attilakreiner Jun 20, 2024
a8a4015
WIP hfs HP
attilakreiner Jun 20, 2024
c2efe21
WIP hfs ignore AppIT
attilakreiner Jun 20, 2024
873b5e1
WIP hfs fix AppIT
attilakreiner Jun 20, 2024
c166431
WIP hfs watchBody 1
attilakreiner Jun 20, 2024
0c0f7f4
WIP hfs watchBody 2
attilakreiner Jun 20, 2024
31b420c
WIP hfs watchBody 3
attilakreiner Jun 21, 2024
d3109f7
WIP hfs watchBody 4
attilakreiner Jun 21, 2024
3198b10
WIP hfs watchBody 5
attilakreiner Jun 21, 2024
ef2d377
WIP hfs watchBody 6
attilakreiner Jun 21, 2024
8f982fe
WIP hfs getPath
attilakreiner Jun 21, 2024
50c8792
WIP hfs watchBody 7
attilakreiner Jun 21, 2024
71231a9
WIP fix
attilakreiner Jun 21, 2024
7535666
Remove BindingConfig.readLocation
jfallows Jun 26, 2024
e7c6c56
No watch event needed for identical response body
jfallows Jun 26, 2024
5423718
Handle status 204 with null body and infer delay for optional prefer …
jfallows Jun 26, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove BindingConfig.readLocation
Remove GuardedConfig.readPath
Rename ConfigAdapterContext.readLocation to ConfigAdapterContext.readResource
Deprecate ConfigAdapterContext for removal
Add EngineConfiguration.configURI() to resolve absolute file path
Remove EngineConfiguration.configURL() but use to default EngineConfiguration.configURI()
Simplify EngineManager
Gather resources on NamespaceConfig from member configs
Consolidate config and resource watcher as EngineConfigWatcher
Resolve watched paths based on path filesystem provider scheme
Configure HttpFileSystem poll internval duration via Map<String,?> env
Simplify ReconfigureHttpIT scripts and include /zilla.yaml in request path
HttpFileSystem per origin (root path) not per individual path
Track HttpPath change count vs read count to implement simple caching
  • Loading branch information
jfallows committed Jun 26, 2024
commit 7535666000f4e806372f80eab69f4ccd97b309a7
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,6 @@ private void attachProxyBinding(
namespaceGenerator.init(binding);
final List<String> labels = configs.stream().map(c -> c.apiLabel).collect(toList());
final NamespaceConfig composite = namespaceGenerator.generateProxy(binding, asyncapis, schemaIdsByApiId::get, labels);
composite.readLocation = binding.readLocation;
attach.accept(composite);
updateNamespace(configs, composite, new ArrayList<>(asyncapis.values()));
}
Expand Down Expand Up @@ -349,7 +348,6 @@ private void attachServerClientBinding(
namespaceConfig.servers.forEach(s -> s.setAsyncapiProtocol(
namespaceGenerator.resolveProtocol(s.protocol(), options, namespaceConfig.asyncapis, namespaceConfig.servers)));
final NamespaceConfig composite = namespaceGenerator.generate(binding, namespaceConfig);
composite.readLocation = binding.readLocation;
attach.accept(composite);
updateNamespace(namespaceConfig.configs, composite, namespaceConfig.asyncapis);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public final class GrpcOptionsConfigAdapter implements OptionsConfigAdapterSpi,

private final GrpcProtobufParser parser = new GrpcProtobufParser();

private Function<String, String> readLocation;
private Function<String, String> readResource;

@Override
public Kind kind()
Expand Down Expand Up @@ -88,7 +88,7 @@ public OptionsConfig adaptFromJson(
public void adaptContext(
ConfigAdapterContext context)
{
this.readLocation = context::readLocation;
this.readResource = context::readResource;
}

private List<GrpcProtobufConfig> asListProtobufs(
Expand All @@ -103,7 +103,7 @@ private GrpcProtobufConfig asProtobuf(
JsonValue value)
{
final String location = ((JsonString) value).getString();
final String protobuf = readLocation.apply(location);
final String protobuf = readResource.apply(location);

return parser.parse(location, protobuf);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public void initJson() throws IOException
{
content = new String(resource.readAllBytes(), UTF_8);
}
Mockito.doReturn(content).when(context).readLocation("protobuf/echo.proto");
Mockito.doReturn(content).when(context).readResource("protobuf/echo.proto");
adapter = new OptionsConfigAdapter(OptionsConfigAdapterSpi.Kind.BINDING, context);
adapter.adaptType("grpc");
JsonbConfig config = new JsonbConfig()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,6 @@ public void attach(
Object2ObjectHashMap::new));

this.composite = namespaceGenerator.generate(binding, openapis, asyncapis, openapiSchemaIdsByApiId::get);
this.composite.readLocation = binding.readLocation;
attach.accept(this.composite);

BindingConfig mappingBinding = composite.bindings.stream()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,6 @@ public void attach(
for (OpenapiNamespaceConfig namespaceConfig : namespaceConfigs.values())
{
final NamespaceConfig composite = namespaceGenerator.generate(binding, namespaceConfig);
composite.readLocation = binding.readLocation;
attach.accept(composite);
namespaceConfig.configs.forEach(c ->
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ public TlsWorker(
.readonly(false)
.build()
.bufferPool();
this.configPath = config.configPath();
this.configPath = Path.of(config.configURI());

this.signaler = new TlsSignaler();

Expand Down
2 changes: 1 addition & 1 deletion runtime/catalog-karapace/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
</licenses>

<properties>
<jacoco.coverage.ratio>0.93</jacoco.coverage.ratio>
<jacoco.coverage.ratio>0.92</jacoco.coverage.ratio>
<jacoco.missed.count>0</jacoco.missed.count>
</properties>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@

import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
Expand Down Expand Up @@ -127,11 +126,9 @@ public void run()

EngineConfiguration config = new EngineConfiguration(props);

URL configURL = config.configURL();
if ("file".equals(configURL.getProtocol()))
Path configPath = Path.of(config.configURI());
if ("file".equals(configPath.getFileSystem().provider().getScheme()))
{
final Path configPath = Paths.get(configURL.getPath());

if (configPath.endsWith("zilla.yaml") && Files.notExists(configPath))
{
Path configJson = configPath.resolveSibling("zilla.json");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
import static org.agrona.LangUtil.rethrowUnchecked;

import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
Expand Down Expand Up @@ -79,7 +77,6 @@ public final class Engine implements Collector, AutoCloseable
private final AtomicInteger nextTaskId;
private final ThreadFactory factory;

private final Path configPath;
private final List<EngineWorker> workers;
private final boolean readonly;
private final EngineConfiguration config;
Expand Down Expand Up @@ -148,7 +145,7 @@ public final class Engine implements Collector, AutoCloseable
for (int workerIndex = 0; workerIndex < workerCount; workerIndex++)
{
EngineWorker worker =
new EngineWorker(config, tasks, labels, errorHandler, tuning::affinity, this::resolvePath, bindings, exporters,
new EngineWorker(config, tasks, labels, errorHandler, tuning::affinity, bindings, exporters,
guards, vaults, catalogs, models, metricGroups, this, this::supplyEventReader,
eventFormatterFactory, workerIndex, readonly, this::process);
workers.add(worker);
Expand Down Expand Up @@ -177,7 +174,6 @@ public final class Engine implements Collector, AutoCloseable
final Map<String, Guard> guardsByType = guards.stream()
.collect(Collectors.toMap(g -> g.name(), g -> g));

this.configPath = config.configPath();
EngineManager manager = new EngineManager(
schemaTypes,
bindingsByType::get,
Expand All @@ -190,10 +186,7 @@ public final class Engine implements Collector, AutoCloseable
logger,
context,
config,
extensions,
this.configPath,
this::readPath,
this::readLocation);
extensions);

this.bindings = bindings;
this.tasks = tasks;
Expand Down Expand Up @@ -285,35 +278,6 @@ public static EngineBuilder builder()
return new EngineBuilder();
}

private String readPath(
Path path)
{
String result;
try
{
//System.out.println("ENG readPath path " + path); // TODO: Ati
result = Files.readString(path);
//System.out.println("ENG readPath result [" + result + "]"); // TODO: Ati
}
catch (Exception ex)
{
result = "";
}
return result;
}

public Path resolvePath(
String location)
{
return configPath.resolveSibling(location);
}

private String readLocation(
String location)
{
return readPath(resolvePath(location));
}

private Thread newTaskThread(
Runnable r)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;

import java.io.File;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.lang.invoke.MethodType;
Expand All @@ -44,7 +45,7 @@ public class EngineConfiguration extends Configuration
public static final boolean DEBUG_BUDGETS = Boolean.getBoolean("zilla.engine.debug.budgets");

public static final PropertyDef<URL> ENGINE_CONFIG_URL;
public static final PropertyDef<Path> ENGINE_CONFIG_PATH;
public static final PropertyDef<URI> ENGINE_CONFIG_URI;
public static final IntPropertyDef ENGINE_CONFIG_POLL_INTERVAL_SECONDS;
public static final PropertyDef<String> ENGINE_NAME;
public static final PropertyDef<String> ENGINE_DIRECTORY;
Expand Down Expand Up @@ -82,8 +83,8 @@ public class EngineConfiguration extends Configuration
{
final ConfigurationDef config = new ConfigurationDef("zilla.engine");
ENGINE_CONFIG_URL = config.property(URL.class, "config.url", EngineConfiguration::configURL, "file:zilla.yaml");
ENGINE_CONFIG_PATH = config.property(Path.class, "config.path", EngineConfiguration::decodeConfigPath,
EngineConfiguration::defaultConfigPath);
ENGINE_CONFIG_URI = config.property(URI.class, "config.uri", EngineConfiguration::decodeConfigURI,
EngineConfiguration::defaultConfigURI);
ENGINE_CONFIG_POLL_INTERVAL_SECONDS = config.property("config.poll.interval.seconds", 60);
ENGINE_NAME = config.property("name", EngineConfiguration::defaultName);
ENGINE_DIRECTORY = config.property("directory", EngineConfiguration::defaultDirectory);
Expand Down Expand Up @@ -144,14 +145,15 @@ public EngineConfiguration()
super(ENGINE_CONFIG, new Configuration());
}

@Deprecated
public URL configURL()
{
return ENGINE_CONFIG_URL.get(this);
}

public Path configPath()
public URI configURI()
{
return ENGINE_CONFIG_PATH.get(this);
return ENGINE_CONFIG_URI.get(this);
}

public int configPollIntervalSeconds()
Expand Down Expand Up @@ -424,39 +426,19 @@ private static HostResolver defaultHostResolver(
};
}

private static Path decodeConfigPath(
private static URI decodeConfigURI(
Configuration config,
String value)
{
return resolveConfigPath(value);
return value.startsWith("file:")
? new File(value.substring("file:".length())).toURI()
: URI.create(value);
}

private static Path defaultConfigPath(
private static URI defaultConfigURI(
Configuration config)
{
URL url = ((EngineConfiguration) config).configURL();
return resolveConfigPath(url.toString());
}

private static Path resolveConfigPath(
String config)
{
Path configPath;
URI configUri = URI.create(config);
if ("file".equals(configUri.getScheme()) && !Path.of(configUri.getSchemeSpecificPart()).isAbsolute())
{
// this works for relative file e.g. file:zilla.yaml
Path basePath = Path.of("").toAbsolutePath();
configPath = basePath.resolve(configUri.getSchemeSpecificPart());
}
else
{
// this works for absolute file e.g. file:/path/dir/zilla.yaml
// this works for http e.g. http://localhost:7115/zilla.yaml
// this works for jar e.g. jar:file:/path/engine.jar!/package/zilla.yaml
// (the jar filesystem is opened and closed by EngineRule)
configPath = Path.of(configUri);
}
return configPath;
URL url = ENGINE_CONFIG_URL.get(config);
return decodeConfigURI(config, url.toString());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ public class BindingConfig
public transient long id;
public transient long entryId;
public transient ToLongFunction<String> resolveId;
public transient Function<String, String> readLocation;

public transient long vaultId;
public transient String qvault;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
*/
package io.aklivity.zilla.runtime.engine.config;

@Deprecated
public interface ConfigAdapterContext
{
String readLocation(String location);
String readResource(
String location);
}
Original file line number Diff line number Diff line change
Expand Up @@ -59,22 +59,19 @@ public final class EngineConfigReader
private final Resolver expressions;
private final Collection<URL> schemaTypes;
private final Consumer<String> logger;
private final Consumer<NamespaceConfig> addResources;

public EngineConfigReader(
EngineConfiguration config,
ConfigAdapterContext context,
Resolver expressions,
Collection<URL> schemaTypes,
Consumer<String> logger,
Consumer<NamespaceConfig> addResources)
Consumer<String> logger)
{
this.config = config;
this.context = context;
this.expressions = expressions;
this.schemaTypes = schemaTypes;
this.logger = logger;
this.addResources = addResources;
}

public EngineConfig read(
Expand Down Expand Up @@ -164,7 +161,6 @@ public EngineConfig read(
reader.reset();
reader.skip(configAt);
NamespaceConfig namespace = jsonb.fromJson(reader, NamespaceConfig.class);
addResources.accept(namespace);
builder.namespace(namespace);

if (!errors.isEmpty())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,9 @@
import static java.util.Objects.requireNonNull;
import static java.util.function.Function.identity;

import java.nio.file.Path;
import java.util.function.Function;

public class GuardConfig
{
public transient long id;
public transient Function<Path, String> readPath;

public final String namespace;
public final String name;
Expand Down
Loading