Skip to content

Commit

Permalink
Fixing issues in deployment, including Netflix#666, Netflix#667 and N…
Browse files Browse the repository at this point in the history 
…etflix#668
  • Loading branch information
@Glamdring
Glamdring committed Mar 19, 2018
1 parent f291f28 commit 588e54b
Show file tree
Hide file tree
Showing 7 changed files with 59 additions and 23 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Expand Up @@ -2,7 +2,7 @@ plugins {
id 'nebula.netflixoss' version '5.0.0'
}

ext.githubProjectName = 'Priam'
ext.githubProjectName = 'Priam-parent'

allprojects {
apply plugin: 'nebula.netflixoss'
Expand Down
16 changes: 12 additions & 4 deletions priam/src/main/java/com/netflix/priam/SimpleDBConfigSource.java
View file
Expand Up @@ -47,15 +47,22 @@
public final class SimpleDBConfigSource extends AbstractConfigSource {
private static final Logger logger = LoggerFactory.getLogger(SimpleDBConfigSource.class.getName());

private static final String DOMAIN = "PriamProperties";
private static String ALL_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s'";
private static final String DEFAULT_DOMAIN = "PriamProperties";
private static String ALL_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s'";

private final Map<String, String> data = Maps.newConcurrentMap();
private final ICredential provider;

private final String domain;

@Inject
public SimpleDBConfigSource(final ICredential provider) {
this.provider = provider;
String configuredDomain = System.getProperty("priam.sdb.properties.domain");
if (configuredDomain == null) {
domain = DEFAULT_DOMAIN;
} else {
domain = configuredDomain;
}
}

@Override
Expand All @@ -68,8 +75,9 @@ public void intialize(final String asgName, final String region) {
String nextToken = null;
String appid = asgName.lastIndexOf('-') > 0 ? asgName.substring(0, asgName.indexOf('-')) : asgName;
logger.info("appid used to fetch properties is: {}", appid);
logger.info("domain used to fetch properties is: {}", domain);
do {
SelectRequest request = new SelectRequest(String.format(ALL_QUERY, appid));
SelectRequest request = new SelectRequest(String.format(ALL_QUERY, domain, appid));
request.setNextToken(nextToken);
SelectResult result = simpleDBClient.select(request);
nextToken = result.getNextToken();
Expand Down
34 changes: 25 additions & 9 deletions priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java
View file
Expand Up @@ -28,11 +28,16 @@

import java.util.*;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* DAO for handling Instance identity information such as token, zone, region
*/
@Singleton
public class SDBInstanceData {
private static final Logger logger = LoggerFactory.getLogger(SDBInstanceData.class.getName());

public static class Attributes {
public final static String APP_ID = "appId";
public final static String ID = "id";
Expand All @@ -45,17 +50,24 @@ public static class Attributes {
public final static String HOSTNAME = "hostname";
}

public static final String DOMAIN = "InstanceIdentity";
public static final String ALL_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s'";
public static final String INSTANCE_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s' and " + Attributes.LOCATION + "='%s' and " + Attributes.ID + "='%d'";
public static final String DEFAULT_DOMAIN = "InstanceIdentity";
public static final String ALL_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s'";
public static final String INSTANCE_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s' and " + Attributes.LOCATION + "='%s' and " + Attributes.ID + "='%d'";

private final ICredential provider;
private final IConfiguration configuration;

private final String domain;

@Inject
public SDBInstanceData(ICredential provider, IConfiguration configuration) {
this.provider = provider;
this.configuration = configuration;
String configuredDomain = System.getProperty("priam.sdb.instanceidentity.domain");
if (configuredDomain == null) {
domain = DEFAULT_DOMAIN;
} else {
domain = configuredDomain;
}
}

/**
Expand All @@ -67,7 +79,9 @@ public SDBInstanceData(ICredential provider, IConfiguration configuration) {
*/
public PriamInstance getInstance(String app, String dc, int id) {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
SelectRequest request = new SelectRequest(String.format(INSTANCE_QUERY, app, dc, id));
String query = String.format(INSTANCE_QUERY, domain, app, dc, id);
logger.info("Fetching instance data using query {}", query);
SelectRequest request = new SelectRequest(query);
SelectResult result = simpleDBClient.select(request);
if (result.getItems().size() == 0)
return null;
Expand All @@ -85,7 +99,9 @@ public Set<PriamInstance> getAllIds(String app) {
Set<PriamInstance> inslist = new HashSet<PriamInstance>();
String nextToken = null;
do {
SelectRequest request = new SelectRequest(String.format(ALL_QUERY, app));
String query = String.format(ALL_QUERY, domain, app);
logger.info("Fetching IDs using query {}", query);
SelectRequest request = new SelectRequest(query);
request.setNextToken(nextToken);
SelectResult result = simpleDBClient.select(request);
nextToken = result.getNextToken();
Expand All @@ -106,7 +122,7 @@ public Set<PriamInstance> getAllIds(String app) {
*/
public void createInstance(PriamInstance instance) throws AmazonServiceException {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
PutAttributesRequest putReq = new PutAttributesRequest(DOMAIN, getKey(instance), createAttributesToRegister(instance));
PutAttributesRequest putReq = new PutAttributesRequest(domain, getKey(instance), createAttributesToRegister(instance));
simpleDBClient.putAttributes(putReq);
}

Expand All @@ -118,7 +134,7 @@ public void createInstance(PriamInstance instance) throws AmazonServiceException
*/
public void registerInstance(PriamInstance instance) throws AmazonServiceException {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
PutAttributesRequest putReq = new PutAttributesRequest(DOMAIN, getKey(instance), createAttributesToRegister(instance));
PutAttributesRequest putReq = new PutAttributesRequest(domain, getKey(instance), createAttributesToRegister(instance));
UpdateCondition expected = new UpdateCondition();
expected.setName(Attributes.INSTANCE_ID);
expected.setExists(false);
Expand All @@ -134,7 +150,7 @@ public void registerInstance(PriamInstance instance) throws AmazonServiceExcepti
*/
public void deregisterInstance(PriamInstance instance) throws AmazonServiceException {
AmazonSimpleDB simpleDBClient = getSimpleDBClient();
DeleteAttributesRequest delReq = new DeleteAttributesRequest(DOMAIN, getKey(instance), createAttributesToDeRegister(instance));
DeleteAttributesRequest delReq = new DeleteAttributesRequest(domain, getKey(instance), createAttributesToDeRegister(instance));
simpleDBClient.deleteAttributes(delReq);
}

Expand Down
4 changes: 2 additions & 2 deletions priam/src/main/java/com/netflix/priam/defaultimpl/PriamConfigSource.java
View file
Expand Up @@ -34,7 +34,7 @@ public PriamConfigSource(final SimpleDBConfigSource simpleDBConfigSource,
// this order was based off PriamConfigurations loading. W/e loaded last could override, but with Composite, first
// has the highest priority.
super(simpleDBConfigSource,
propertiesConfigSource,
systemPropertiesConfigSource);
systemPropertiesConfigSource,
propertiesConfigSource);
}
}
3 changes: 2 additions & 1 deletion priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java
View file
Expand Up @@ -26,6 +26,7 @@
import com.netflix.priam.aws.auth.EC2RoleAssumptionCredential;
import com.netflix.priam.aws.auth.IS3Credential;
import com.netflix.priam.aws.auth.S3RoleAssumptionCredential;
import com.netflix.priam.aws.IAMCredential;
import com.netflix.priam.backup.BackupFileSystemContext;
import com.netflix.priam.backup.IBackupFileSystem;
import com.netflix.priam.backup.IBackupMetrics;
Expand Down Expand Up @@ -63,7 +64,7 @@ protected void configure() {
bind(IFileCryptography.class).annotatedWith(Names.named("filecryptoalgorithm")).to(PgpCryptography.class);
bind(ICredentialGeneric.class).annotatedWith(Names.named("gcscredential")).to(GcsCredential.class);
bind(ICredentialGeneric.class).annotatedWith(Names.named("pgpcredential")).to(PgpCredential.class);
bind(ICredential.class).to(ClearCredential.class);
bind(ICredential.class).to(IAMCredential.class);
bind(IDeadTokenRetriever.class).to(DeadTokenRetriever.class);
bind(IPreGeneratedTokenRetriever.class).to(PreGeneratedTokenRetriever.class);
bind(INewTokenRetriever.class).to(NewTokenRetriever.class);
Expand Down
6 changes: 4 additions & 2 deletions priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java
View file
Expand Up @@ -57,8 +57,10 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed
map.put("rpc_port", config.getThriftPort());
map.put("start_native_transport", config.isNativeTransportEnabled());
map.put("native_transport_port", config.getNativeTransportPort());
map.put("listen_address", hostname);
map.put("rpc_address", hostname);
if (hostname != null) {
map.put("listen_address", hostname);
map.put("rpc_address", hostname);
}
//Dont bootstrap in restore mode
if (!Restore.isRestoreEnabled(config)) {
map.put("auto_bootstrap", config.getAutoBoostrap());
Expand Down
17 changes: 13 additions & 4 deletions priam/src/main/resources/Priam.properties
View file
Expand Up @@ -10,19 +10,27 @@ priam.backup.retention=
priam.backup.threads=2
priam.bootcluster=
priam.cache.location=/var/lib/cassandra/saved_caches
priam.cass.home=/mnt/cassandra
priam.cass.home=/etc/cassandra
priam.cass.manual.start.enable=
priam.cass.process=
priam.cass.startscript=/mnt/cassandra/bin/cassandra
priam.cass.stopscript=/mnt/cassandra/bin/cassandra
priam.cass.startscript=/etc/init.d/cassandra start
priam.cass.stopscript=/etc/init.d/cassandra stop
priam.clustername=cass_cluster
priam.commitlog.location=/var/lib/cassandra/commitlog
priam.compaction.throughput=
priam.data.location=/var/lib/cassandra/data
priam.direct.memory.size.m1.large=1G
priam.endpoint_snitch=org.apache.cassandra.locator.Ec2Snitch
priam.heap.newgen.size.m1.large=2G
priam.heap.size.m1.large=4G
priam.heap.size.m1.large=1G
priam.heap.newgen.size.t2.small=1G
priam.heap.size.t2.small=1G
priam.heap.newgen.size.t2.micro=512M
priam.heap.size.t2.micro=512M
priam.heap.newgen.size.m3.medium=1G
priam.heap.size.m3.medium=1G
priam.heap.newgen.size.m3.large=1G
priam.heap.size.m3.large=1G
priam.hint.delay=
priam.hint.window=
priam.jmx.port=7199
Expand Down Expand Up @@ -53,3 +61,4 @@ priam.thrift.port=9160
priam.upload.throttle=
priam.yamlLocation=
priam.zones.available=
priam.nativeTransport.enabled=true

0 comments on commit 588e54b

Please sign in to comment.