Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixing issues in deployment, including #666, #667 and #668 #670

Open
wants to merge 5 commits into
base: 3.11
Choose a base branch
from
Open

Fixing issues in deployment, including #666, #667 and #668 #670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
588e54b
Fixing issues in deployment, including #666, #667 and #668
Glamdring Mar 19, 2018
ad61c31
Moving instanceidentity domain configuration to the IConfiguration in…
Glamdring Mar 22, 2018
98651a9
fixing prefix handling for S3 buckets
Glamdring May 5, 2018
5791842
fallback to old code in case of missing lifecycle filter
Glamdring May 5, 2018
aa44600
fixes
Glamdring Jan 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.gradle
View file
Open in desktop
Expand Up @@ -2,7 +2,7 @@ plugins {
id 'nebula.netflixoss' version '5.0.0'
}

ext.githubProjectName = 'Priam'
ext.githubProjectName = 'Priam-parent'

allprojects {
apply plugin: 'nebula.netflixoss'
Expand Down
6 changes: 5 additions & 1 deletion priam/src/main/java/com/netflix/priam/IConfiguration.java
View file
Open in desktop
Expand Up @@ -26,7 +26,6 @@

import java.util.List;
import java.util.Map;
import java.util.Set;

/**
* Interface for Priam's configuration
Expand Down Expand Up @@ -739,4 +738,9 @@ public interface IConfiguration {
* @return SNS Topic ARN to be used to send notification.
*/
public String getBackupNotificationTopicArn();

/**
* @return the SimpleDB domain name for storing instance identities
*/
public String getInstanceIdentityDomain();
}
16 changes: 12 additions & 4 deletions priam/src/main/java/com/netflix/priam/SimpleDBConfigSource.java
View file
Open in desktop
Expand Up @@ -47,15 +47,22 @@
public final class SimpleDBConfigSource extends AbstractConfigSource {
private static final Logger logger = LoggerFactory.getLogger(SimpleDBConfigSource.class.getName());

private static final String DOMAIN = "PriamProperties";
private static String ALL_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s'";
private static final String DEFAULT_DOMAIN = "PriamProperties";
private static String ALL_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s'";

private final Map<String, String> data = Maps.newConcurrentMap();
private final ICredential provider;

private final String domain;

@Inject
public SimpleDBConfigSource(final ICredential provider) {
this.provider = provider;
String configuredDomain = System.getProperty("priam.sdb.properties.domain");
if (configuredDomain == null) {
domain = DEFAULT_DOMAIN;
} else {
domain = configuredDomain;
}
}

@Override
Expand All @@ -68,8 +75,9 @@ public void intialize(final String asgName, final String region) {
String nextToken = null;
String appid = asgName.lastIndexOf('-') > 0 ? asgName.substring(0, asgName.indexOf('-')) : asgName;
logger.info("appid used to fetch properties is: {}", appid);
logger.info("domain used to fetch properties is: {}", domain);
do {
SelectRequest request = new SelectRequest(String.format(ALL_QUERY, appid));
SelectRequest request = new SelectRequest(String.format(ALL_QUERY, domain, appid));
request.setNextToken(nextToken);
SelectResult result = simpleDBClient.select(request);
nextToken = result.getNextToken();
Expand Down
86 changes: 64 additions & 22 deletions priam/src/main/java/com/netflix/priam/aws/S3FileSystemBase.java
View file
Open in desktop
Expand Up @@ -15,11 +15,32 @@
*/
package com.netflix.priam.aws;

import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.BucketLifecycleConfiguration;
import com.amazonaws.services.s3.model.BucketLifecycleConfiguration.Rule;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.lifecycle.LifecycleAndOperator;
import com.amazonaws.services.s3.model.lifecycle.LifecycleFilter;
import com.amazonaws.services.s3.model.lifecycle.LifecyclePredicateVisitor;
import com.amazonaws.services.s3.model.lifecycle.LifecyclePrefixPredicate;
import com.amazonaws.services.s3.model.lifecycle.LifecycleTagPredicate;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.RateLimiter;
import com.google.inject.Provider;
Expand All @@ -38,21 +59,6 @@
import com.netflix.priam.notification.EventGenerator;
import com.netflix.priam.notification.EventObserver;
import com.netflix.priam.scheduler.BlockingSubmitThreadPoolExecutor;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

public abstract class S3FileSystemBase implements IBackupFileSystem, EventGenerator<BackupEvent> {
protected static final int MAX_CHUNKS = 10000;
Expand Down Expand Up @@ -151,10 +157,19 @@ public void cleanup() {

private boolean updateLifecycleRule(IConfiguration config, List<Rule> rules, String prefix) {
Rule rule = null;
PrefixVisitor visitor = new PrefixVisitor(prefix);
for (BucketLifecycleConfiguration.Rule lcRule : rules) {
if (lcRule.getPrefix().equals(prefix)) {
rule = lcRule;
break;
if (lcRule.getFilter() != null) {
lcRule.getFilter().getPredicate().accept(visitor);
if (visitor.isMatchesPrefix()) {
rule = lcRule;
break;
}
} else if (lcRule.getPrefix() != null) {
if (lcRule.getPrefix().equals(prefix)) {
rule = lcRule;
break;
}
}
}
if (rule == null && config.getBackupRetentionDays() <= 0)
Expand All @@ -165,21 +180,48 @@ private boolean updateLifecycleRule(IConfiguration config, List<Rule> rules, Str
}
if (rule == null) {
// Create a new rule
rule = new BucketLifecycleConfiguration.Rule().withExpirationInDays(config.getBackupRetentionDays()).withPrefix(prefix);
rule = new BucketLifecycleConfiguration.Rule().withExpirationInDays(config.getBackupRetentionDays())
.withFilter(new LifecycleFilter(new LifecyclePrefixPredicate(prefix)));
rule.setStatus(BucketLifecycleConfiguration.ENABLED);
rule.setId(prefix);
rules.add(rule);
logger.info("Setting cleanup for {} to {} days", rule.getPrefix(), rule.getExpirationInDays());
logger.info("Setting cleanup for {} to {} days", prefix, rule.getExpirationInDays());
} else if (config.getBackupRetentionDays() > 0) {
logger.info("Setting cleanup for {} to {} days", rule.getPrefix(), config.getBackupRetentionDays());
logger.info("Setting cleanup for {} to {} days", prefix, config.getBackupRetentionDays());
rule.setExpirationInDays(config.getBackupRetentionDays());
} else {
logger.info("Removing cleanup rule for {}", rule.getPrefix());
logger.info("Removing cleanup rule for {}", prefix);
rules.remove(rule);
}
return true;
}


private class PrefixVisitor implements LifecyclePredicateVisitor {
private String prefix;
private boolean matchesPrefix;

public PrefixVisitor(String prefix) {
this.prefix = prefix;
}

@Override
public void visit(LifecycleAndOperator lifecycleAndOperator) {
}
@Override
public void visit(LifecycleTagPredicate lifecycleTagPredicate) {
}
@Override
public void visit(LifecyclePrefixPredicate lifecyclePrefixPredicate) {
if (lifecyclePrefixPredicate.getPrefix().equals(prefix)) {
matchesPrefix = true;
}
}

public boolean isMatchesPrefix() {
return matchesPrefix;
}
}
/*
@param path - representation of the file uploaded
@param start time of upload, in millisecs
Expand Down
33 changes: 22 additions & 11 deletions priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java
View file
Open in desktop
Expand Up @@ -28,11 +28,17 @@

import java.util.*;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* DAO for handling Instance identity information such as token, zone, region
*/
@Singleton
public class SDBInstanceData {
private static final Logger logger = LoggerFactory.getLogger(SDBInstanceData.class.getName());

public static class Attributes {
public final static String APP_ID = "appId";
public final static String ID = "id";
Expand All @@ -45,17 +51,18 @@ public static class Attributes {
public final static String HOSTNAME = "hostname";
}

public static final String DOMAIN = "InstanceIdentity";
public static final String ALL_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s'";
public static final String INSTANCE_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s' and " + Attributes.LOCATION + "='%s' and " + Attributes.ID + "='%d'";
public static final String ALL_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s'";
public static final String INSTANCE_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s' and " + Attributes.LOCATION + "='%s' and " + Attributes.ID + "='%d'";

private final ICredential provider;
private final IConfiguration configuration;

private final String domain;

@Inject
public SDBInstanceData(ICredential provider, IConfiguration configuration) {
this.provider = provider;
this.configuration = configuration;
this.domain = configuration.getInstanceIdentityDomain();
}

/**
Expand All @@ -67,7 +74,9 @@ public SDBInstanceData(ICredential provider, IConfiguration configuration) {
*/
public PriamInstance getInstance(String app, String dc, int id) {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
SelectRequest request = new SelectRequest(String.format(INSTANCE_QUERY, app, dc, id));
String query = String.format(INSTANCE_QUERY, domain, app, dc, id);
logger.info("Fetching instance data using query {}", query);
SelectRequest request = new SelectRequest(query);
SelectResult result = simpleDBClient.select(request);
if (result.getItems().size() == 0)
return null;
Expand All @@ -85,7 +94,9 @@ public Set<PriamInstance> getAllIds(String app) {
Set<PriamInstance> inslist = new HashSet<PriamInstance>();
String nextToken = null;
do {
SelectRequest request = new SelectRequest(String.format(ALL_QUERY, app));
String query = String.format(ALL_QUERY, domain, app);
logger.info("Fetching IDs using query {}", query);
SelectRequest request = new SelectRequest(query);
request.setNextToken(nextToken);
SelectResult result = simpleDBClient.select(request);
nextToken = result.getNextToken();
Expand All @@ -106,7 +117,7 @@ public Set<PriamInstance> getAllIds(String app) {
*/
public void createInstance(PriamInstance instance) throws AmazonServiceException {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
PutAttributesRequest putReq = new PutAttributesRequest(DOMAIN, getKey(instance), createAttributesToRegister(instance));
PutAttributesRequest putReq = new PutAttributesRequest(domain, getKey(instance), createAttributesToRegister(instance));
simpleDBClient.putAttributes(putReq);
}

Expand All @@ -118,7 +129,7 @@ public void createInstance(PriamInstance instance) throws AmazonServiceException
*/
public void registerInstance(PriamInstance instance) throws AmazonServiceException {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
PutAttributesRequest putReq = new PutAttributesRequest(DOMAIN, getKey(instance), createAttributesToRegister(instance));
PutAttributesRequest putReq = new PutAttributesRequest(domain, getKey(instance), createAttributesToRegister(instance));
UpdateCondition expected = new UpdateCondition();
expected.setName(Attributes.INSTANCE_ID);
expected.setExists(false);
Expand All @@ -134,15 +145,15 @@ public void registerInstance(PriamInstance instance) throws AmazonServiceExcepti
*/
public void deregisterInstance(PriamInstance instance) throws AmazonServiceException {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
DeleteAttributesRequest delReq = new DeleteAttributesRequest(DOMAIN, getKey(instance), createAttributesToDeRegister(instance));
DeleteAttributesRequest delReq = new DeleteAttributesRequest(domain, getKey(instance), createAttributesToDeRegister(instance));
simpleDBClient.deleteAttributes(delReq);
}

protected List<ReplaceableAttribute> createAttributesToRegister(PriamInstance instance) {
instance.setUpdatetime(new Date().getTime());
List<ReplaceableAttribute> attrs = new ArrayList<ReplaceableAttribute>();
attrs.add(new ReplaceableAttribute(Attributes.INSTANCE_ID, instance.getInstanceId(), false));
attrs.add(new ReplaceableAttribute(Attributes.TOKEN, instance.getToken(), true));
attrs.add(new ReplaceableAttribute(Attributes.TOKEN, StringUtils.trimToEmpty(instance.getToken()), true));
attrs.add(new ReplaceableAttribute(Attributes.APP_ID, instance.getApp(), true));
attrs.add(new ReplaceableAttribute(Attributes.ID, Integer.toString(instance.getId()), true));
attrs.add(new ReplaceableAttribute(Attributes.AVAILABILITY_ZONE, instance.getRac(), true));
Expand All @@ -156,7 +167,7 @@ protected List<ReplaceableAttribute> createAttributesToRegister(PriamInstance in
protected List<Attribute> createAttributesToDeRegister(PriamInstance instance) {
List<Attribute> attrs = new ArrayList<Attribute>();
attrs.add(new Attribute(Attributes.INSTANCE_ID, instance.getInstanceId()));
attrs.add(new Attribute(Attributes.TOKEN, instance.getToken()));
attrs.add(new Attribute(Attributes.TOKEN, StringUtils.trimToEmpty(instance.getToken())));
attrs.add(new Attribute(Attributes.APP_ID, instance.getApp()));
attrs.add(new Attribute(Attributes.ID, Integer.toString(instance.getId())));
attrs.add(new Attribute(Attributes.AVAILABILITY_ZONE, instance.getRac()));
Expand Down
4 changes: 2 additions & 2 deletions priam/src/main/java/com/netflix/priam/defaultimpl/PriamConfigSource.java
View file
Open in desktop
Expand Up @@ -34,7 +34,7 @@ public PriamConfigSource(final SimpleDBConfigSource simpleDBConfigSource,
// this order was based off PriamConfigurations loading. W/e loaded last could override, but with Composite, first
// has the highest priority.
super(simpleDBConfigSource,
propertiesConfigSource,
systemPropertiesConfigSource);
systemPropertiesConfigSource,
propertiesConfigSource);
}
}
7 changes: 7 additions & 0 deletions priam/src/main/java/com/netflix/priam/defaultimpl/PriamConfiguration.java
View file
Open in desktop
Expand Up @@ -184,6 +184,7 @@ public class PriamConfiguration implements IConfiguration {
private static final String CONFIG_VPC_ROLE_ASSUMPTION_ARN = PRIAM_PRE + ".vpc.roleassumption.arn";
private static final String CONFIG_DUAL_ACCOUNT = PRIAM_PRE + ".roleassumption.dualaccount";

private static final String CONFIG_INSTANCE_IDENTITY_DOMAIN = PRIAM_PRE + ".sdb.instanceidentity.domain";

//Running instance meta data
private String RAC;
Expand Down Expand Up @@ -245,6 +246,8 @@ public class PriamConfiguration implements IConfiguration {
private static final int DEFAULT_TOMBSTONE_WARNING_THRESHOLD = 1000; // C* defaults
private static final int DEFAULT_TOMBSTONE_FAILURE_THRESHOLD = 100000;// C* defaults

private static final String DEFAULT_INSTANCE_IDENTITY_DOMAIN = "InstanceIdentity";

// AWS EC2 Dual Account
private static final boolean DEFAULT_DUAL_ACCOUNT = false;

Expand Down Expand Up @@ -1127,4 +1130,8 @@ public String getBackupNotificationTopicArn() {
return config.get(PRIAM_PRE + ".backup.notification.topic.arn", "");
}

@Override
public String getInstanceIdentityDomain() {
return config.get(CONFIG_INSTANCE_IDENTITY_DOMAIN, DEFAULT_INSTANCE_IDENTITY_DOMAIN);
}
}
3 changes: 2 additions & 1 deletion priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java
View file
Open in desktop
Expand Up @@ -26,6 +26,7 @@
import com.netflix.priam.aws.auth.EC2RoleAssumptionCredential;
import com.netflix.priam.aws.auth.IS3Credential;
import com.netflix.priam.aws.auth.S3RoleAssumptionCredential;
import com.netflix.priam.aws.IAMCredential;
import com.netflix.priam.backup.BackupFileSystemContext;
import com.netflix.priam.backup.IBackupFileSystem;
import com.netflix.priam.backup.IBackupMetrics;
Expand Down Expand Up @@ -63,7 +64,7 @@ protected void configure() {
bind(IFileCryptography.class).annotatedWith(Names.named("filecryptoalgorithm")).to(PgpCryptography.class);
bind(ICredentialGeneric.class).annotatedWith(Names.named("gcscredential")).to(GcsCredential.class);
bind(ICredentialGeneric.class).annotatedWith(Names.named("pgpcredential")).to(PgpCredential.class);
bind(ICredential.class).to(ClearCredential.class);
bind(ICredential.class).to(IAMCredential.class);
bind(IDeadTokenRetriever.class).to(DeadTokenRetriever.class);
bind(IPreGeneratedTokenRetriever.class).to(PreGeneratedTokenRetriever.class);
bind(INewTokenRetriever.class).to(NewTokenRetriever.class);
Expand Down
6 changes: 4 additions & 2 deletions priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java
View file
Open in desktop
Expand Up @@ -57,8 +57,10 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed
map.put("rpc_port", config.getThriftPort());
map.put("start_native_transport", config.isNativeTransportEnabled());
map.put("native_transport_port", config.getNativeTransportPort());
map.put("listen_address", hostname);
map.put("rpc_address", hostname);
if (hostname != null) {
map.put("listen_address", hostname);
map.put("rpc_address", hostname);
}
//Dont bootstrap in restore mode
if (!Restore.isRestoreEnabled(config)) {
map.put("auto_bootstrap", config.getAutoBoostrap());
Expand Down