low_cardinality_cols = [cname for cname in candidate_train_predictors.columns if 
                                candidate_train_predictors[cname].nunique() < 10 and
                                candidate_train_predictors[cname].dtype == "object"]
numeric_cols = [cname for cname in candidate_train_predictors.columns if 
                                candidate_train_predictors[cname].dtype in ['int64', 'float64']]

train_predictors.dtypes.sample(10)

LandContour     object
ScreenPorch      int64
BsmtFinSF2       int64
SaleType        object
BedroomAbvGr     int64
(snip)

one_hot_encoded_training_predictors = pd.get_dummies(train_predictors)

one_hot_encoded_training_predictors = pd.get_dummies(train_predictors)
one_hot_encoded_test_predictors = pd.get_dummies(test_predictors)
final_train, final_test = one_hot_encoded_training_predictors.align(one_hot_encoded_test_predictors,
                                                                    join='left', 
                                                                    axis=1)

cols_with_missing = [col for col in X_train.columns 
                                 if X_train[col].isnull().any()]
reduced_X_train = X_train.drop(cols_with_missing, axis=1)
reduced_X_test  = X_test.drop(cols_with_missing, axis=1)

from sklearn.impute import SimpleImputer

my_imputer = SimpleImputer()
imputed_X_train = my_imputer.fit_transform(X_train)
imputed_X_test = my_imputer.transform(X_test)

imputed_X_train_plus = X_train.copy()
imputed_X_test_plus = X_test.copy()

cols_with_missing = (col for col in X_train.columns 
                                 if X_train[col].isnull().any())
for col in cols_with_missing:
    imputed_X_train_plus[col + '_was_missing'] = imputed_X_train_plus[col].isnull()
    imputed_X_test_plus[col + '_was_missing'] = imputed_X_test_plus[col].isnull()

$ conda install nltk

import nltk
nltk.download("stopwords")

symbol = ["'", '"', ':', ';', '.', ',', '-', '!', '?', "'s"]
words_wo_stopwords = [w.lower() for w in all_words if not w in stop_words + symbol]

clean_frequency = nltk.FreqDist(words_wo_stopwords)

plt.figure(figsize=(10, 7))
clean_frequency.plot(30,cumulative=True)

Keras==2.2.0
tensorflow==1.8.0
numpy==1.13.3

linear_regression
├── model.h5
├── model.py
├── requirements.txt
├── serving.yaml
└── src
    └── func_main.py

$ curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{
"x": [[1, 1],[1, 1]]}' 'http://localhost/gateway/applications/linear_regression/infer'

$ mkdir HydroServing
$ cd HydroServing/
$ git clone https://github.com/Hydrospheredata/hydro-serving
$ hydro-serving
$ sudo docker-compose up -d

$ sudo docker-compose ps
  Name                 Command               State                        Ports
-----------------------------------------------------------------------------------------------------
gateway     /hydro-serving/app/start.sh      Up      0.0.0.0:29090->9090/tcp, 0.0.0.0:29091->9091/tcp
manager     /hydro-serving/app/start.sh      Up      0.0.0.0:19091->9091/tcp
managerui   /bin/sh -c envsubst '${MAN ...   Up      80/tcp, 9091/tcp
postgres    docker-entrypoint.sh postgres    Up
sidecar     /hydro-serving/start.sh          Up      0.0.0.0:80->8080/tcp, 0.0.0.0:8082->8082/tcp

$ conda create -n hydro-serving python=3.6 python
$ source activate hydro-serving
$ conda install keras scikit-learn
$ pip install hs

$ hs cluster add --name local --server http://localhost
--- Logging error ---
Traceback (most recent call last):
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/logging/__init__.py", line 992, in emit
    msg = self.format(record)
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/logging/__init__.py", line 838, in format
    return fmt.format(record)
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/logging/__init__.py", line 575, in format
    record.message = record.getMessage()
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/logging/__init__.py", line 338, in getMessage
    msg = msg % self.args
TypeError: not all arguments converted during string formatting
Call stack:
  File "/home/centos/.conda/envs/hydro-serving/bin/hs", line 11, in <module>
    sys.exit(hs_cli())
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/site-packages/click/core.py", line 722, in __call__
    return self.main(*args, **kwargs)
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/site-packages/click/core.py", line 697, in main
    rv = self.invoke(ctx)
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/site-packages/click/core.py", line 1063, in invoke
    Command.invoke(self, ctx)
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/site-packages/click/core.py", line 895, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/site-packages/click/core.py", line 535, in invoke
    return callback(*args, **kwargs)
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/site-packages/click/decorators.py", line 17, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/site-packages/hydroserving/cli/hs.py", line 18, in hs_cli
    ctx.obj.services = ContextServices.with_config_path(HOME_PATH_EXPANDED)
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/site-packages/hydroserving/models/context_object.py", line 44, in with_config_path
    config_service = ConfigService(path)
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/site-packages/hydroserving/services/config.py", line 17, in __init__
    logging.error("{} is not an existing directory", home_path)
Message: '{} is not an existing directory'
Arguments: ('/home/centos/.hs-home',)
WARNING:root:Using local as current cluster
Cluster 'local' @ http://localhost added successfully

$ hs cluster
Current cluster: {'cluster': {'server': 'http://localhost'}, 'name': 'local'}

$ mkdir -p ~/Sources/linear_regression
$ cd ~/Sources/linear_regression
$ cat << EOF > model.py
  from keras.models import Sequential
  from keras.layers import Dense
  from sklearn.datasets import make_regression
  from sklearn.model_selection import train_test_split
  from sklearn.preprocessing import MinMaxScaler
  
  # initialize data
  n_samples = 1000
  X, y = make_regression(n_samples=n_samples, n_features=2, noise=0.5, random_state=112)
  
  scallar_x, scallar_y = MinMaxScaler(), MinMaxScaler()
  scallar_x.fit(X)
  scallar_y.fit(y.reshape(n_samples, 1))
  X = scallar_x.transform(X)
  y = scallar_y.transform(y.reshape(n_samples, 1))
  
  # create a model
  model = Sequential()
  model.add(Dense(4, input_dim=2, activation='relu'))
  model.add(Dense(4, activation='relu'))
  model.add(Dense(1, activation='linear'))
  
  model.compile(loss='mse', optimizer='adam')
  model.fit(X, y, epochs=100)
  
  # save model
  model.save('model.h5')
  EOF

$ python model.py

$ mkdir src
$ cd src
$ cat << EOF > func_main.py
  import numpy as np
  import hydro_serving_grpc as hs
  from keras.models import load_model
  
  # 0. Load model once
  model = load_model('/model/files/model.h5')
  
  def infer(x):
      # 1. Retrieve tensor's content and put it to numpy array
      data = np.array(x.double_val)
      data = data.reshape([dim.size for dim in x.tensor_shape.dim])
  
      # 2. Make a prediction
      result = model.predict(data)
      
      # 3. Pack the answer
      y_shape = hs.TensorShapeProto(dim=[hs.TensorShapeProto.Dim(size=-1)])
      y_tensor = hs.TensorProto(
          dtype=hs.DT_DOUBLE,
          double_val=result.flatten(),
          tensor_shape=y_shape)
  
      # 4. Return the result
      return hs.PredictResponse(outputs={"y": y_tensor})
  EOF

$ cd ..
$ cat << EOF > serving.yaml
  kind: Model
  name: linear_regression
  model-type: python:3.6
  payload:
    - "src/"
    - "requirements.txt"
    - "model.h5"
  
  contract:
    infer:                    # Signature function
      inputs:
        x:                    # Input field
          shape: [-1, 2]
          type: double
          profile: numerical
      outputs:
        y:                    # Output field
          shape: [-1]
          type: double
          profile: numerical
  EOF

$ cat << EOF > requirements.txt
  Keras==2.2.0
  tensorflow==1.8.0
  numpy==1.13.3
  EOF

$ tree
.
├── model.h5
├── model.py
├── requirements.txt
├── serving.yaml
└── src
    └── func_main.py

1 directory, 5 files

$ hs upload

Using 'local' cluster
['/home/centos/Sources/linear_regression/src', '/home/centos/Sources/linear_regression/requirements.txt', '/home/centos/Sources/linear_regression/model.h5']
Packing the model  [####################################]  100%
Assembling the model  [####################################]  100%
Uploading to http://localhost
Uploading model assembly  [####################################]  100%
Traceback (most recent call last):
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/site-packages/hydroserving/httpclient/remote_connection.py", line 82, in postprocess_response
    response.raise_for_status()
  File "/home/centos/.conda/envs/hydro-serving/lib/python3.6/site-packages/requests/models.py", line 940, in raise_for_status
    raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http://localhost/api/v1/model/upload

During handling of the above exception, another exception occurred:

(snip)

$ pip uninstall hs
$ pip install hs==2.0.0rc2

runtime: "hydrosphere/serving-runtime-python:3.6-latest"

$ curl -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' -d '{   "x": [     [       1,       1     ]   ] }' 'http://10.0.0.209/gateway/application/linear_regression'

{"error":"InternalUncaught","information":"UNKNOWN: Exception calling application: No module named 'numpy'"}

$ sudo docker run -p 2004:2004 \
   -v /var/run/docker.sock:/var/run/docker.sock \
   hydrosphere/mist:1.1.1-2.3.0

$ mkdir ~/Spark
$ cd ~/Spark
$ wget http://ftp.riken.jp/net/apache/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.7.tgz
$ tar xvzf spark-2.3.3-bin-hadoop2.7.tgz
$ ln -s spark-2.3.3-bin-hadoop2.7 default

$ mkdir ~/HydroMist
$ cd ~/HydroMist
$ wget http://repo.hydrosphere.io/hydrosphere/static/mist-1.1.1.tar.gz
$ tar xvfz mist-1.1.1.tar.gz
$ ln -s mist-1.1.1 default
$ cd default

$ SPARK_HOME=${HOME}/Spark/default ./bin/mist-master start --debug true

$ conda create -n mist python=3.6 python
$ source activate mist
$ pip install mist-cli

$ mkdir -p ~/Sources
$ cd ~/Sources
$ git clone https://github.com/Hydrospheredata/hello_mist.git

$ cd hello_mist/scala
$ sbt package
$ mist-cli apply -f conf -u ''

Process 5 file entries
updating Artifact hello-mist-scala_0.0.1.jar
Success: Artifact hello-mist-scala_0.0.1.jar
updating Context emr_ctx
Success: Context emr_ctx
updating Context emr_autoscale_ctx
Success: Context emr_autoscale_ctx
updating Context standalone
Success: Context standalone
updating Function hello-mist-scala
Success: Function hello-mist-scala


Get context info
--------------------------------------------------------------------------------
curl -H 'Content-Type: application/json' -X POST http://localhost:2004/v2/api/contexts/emr_ctx


Get context info
--------------------------------------------------------------------------------
curl -H 'Content-Type: application/json' -X POST http://localhost:2004/v2/api/contexts/emr_autoscale_ctx


Get context info
--------------------------------------------------------------------------------
curl -H 'Content-Type: application/json' -X POST http://localhost:2004/v2/api/contexts/standalone


Get info of function resource
--------------------------------------------------------------------------------
curl  -H 'Content-Type: application/json' -X GET http://localhost:2004/v2/api/functions/hello-mist-scala

Start job via mist-cli
--------------------------------------------------------------------------------
mist-cli --host localhost --port 2004 start job hello-mist-scala '{"samples": 7}'

Start job via curl
--------------------------------------------------------------------------------
curl --data '{"samples": 7}' -H 'Content-Type: application/json' -X POST http://localhost:2004/v2/api/functions/hello-mist-scala/jobs?force=true

$ curl -H 'Content-Type: application/json' -X GET http://localhost:2004/v2/api/contexts/standalone | jq
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   252  100   252    0     0  26537      0 --:--:-- --:--:-- --:--:-- 28000
{
  "name": "standalone",
  "maxJobs": 1,
  "maxConnFailures": 1,
  "workerMode": "exclusive",
  "precreated": false,
  "sparkConf": {
    "spark.submit.deployMode": "cluster",
    "spark.master": "spark://holy-excalibur:7077"
  },
  "runOptions": "",
  "downtime": "1200s",
  "streamingDuration": "1s"
}

import mist.api._
import mist.api.dsl._
import mist.api.encoding.defaults._
import org.apache.spark.SparkContext

object HelloMist extends MistFn with Logging {

  override def handle = {
    withArgs(
      arg[Int]("samples", 10000)
    )
    .withMistExtras
    .onSparkContext((n: Int, extras: MistExtras, sc: SparkContext) => {
      import extras._

      logger.info(s"Hello Mist started with samples: $n")

      val count = sc.parallelize(1 to n).filter(_ => {
        val x = math.random
        val y = math.random
        x * x + y * y < 1
      }).count()

      val pi = (4.0 * count) / n
      pi
    }).asHandle
  }
}

$ mist-cli list contexts
       ID           WORKER MODE
default             exclusiveemr_ctx             exclusiveemr_autoscale_ctx   exclusivestandalone          exclusive

$ mist-cli list functions
    FUNCTION       DEFAULT CONTEXT              PATH              CLASS NAME
hello-mist-scala   default           hello-mist-scala_0.0.1.jar   HelloMist$

$ mist-cli status
Mist version: 1.1.1
Spark version: 2.4.0
Java version: 1.8.0_201-b09

object PiExample extends MistFn {

  override def handle: Handle = {
    val samples = arg[Int]("samples").validated(_ > 0, "Samples should be positive")

(snip)

val samples = arg[Int]("samples").validated(_ > 0, "Samples should be positive")

val three = withArgs(arg[Int]("n"), arg[String]("str"), arg[Boolean]("flag"))

val three = arg[Int]("n") & arg[String]("str") & arg[Boolean]("flag")

def onSparkSession[F, Cmb, Out](f: F)(
  implicit
  cmb: ArgCombiner.Aux[A, SparkSession, Cmb],
  fnT: FnForTuple.Aux[Cmb, F, Out]
): RawHandle[Out] = args.combine(sparkSessionArg).apply(f)

    withArgs(samples).onSparkContext((n: Int, sc: SparkContext) => {
      val count = sc.parallelize(1 to n).filter(_ => {

(snip)

import org.apache.spark.sql.SparkSession

(snip)

    withArgs(samples).onSparkSession((n: Int, spark: SparkSession) => {
      val count = spark.sparkContext.parallelize(1 to n).filter(_ => {

(snip)

def onSparkContext[F, Cmb, Out](f: F)(
  implicit
  cmb: ArgCombiner.Aux[A, SparkContext, Cmb],
  fnT: FnForTuple.Aux[Cmb, F, Out]
): RawHandle[Out] = args.combine(sparkContextArg).apply(f)

withArgs(samples).onSparkContext((n: Int, sc: SparkContext) => {

(snip)

}).asHandle

implicit class AsHandleOps[A](raw: RawHandle[A]) {
  def asHandle(implicit enc: JsEncoder[A]): Handle = raw.toHandle(enc)
}

$ curl -X GET 'http://10.0.0.209:2004/v2/api/functions' | jq
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   208  100   208    0     0  13256      0 --:--:-- --:--:-- --:--:-- 13866
[
  {
    "name": "hello-mist-scala",
    "execute": {
      "samples": {
        "type": "MOption",
        "args": [
          {
            "type": "MInt"
          }
        ]
      }
    },
    "path": "hello-mist-scala_0.0.1.jar",
    "tags": [],
    "className": "HelloMist$",
    "defaultContext": "default",
    "lang": "scala"
  }
]

$ curl -X GET 'http://10.0.0.209:2004/v2/api/functions/hello-mist-scala/jobs' | jq
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  1682  100  1682    0     0  29484      0 --:--:-- --:--:-- --:--:-- 30035
[
  {
    "source": "Http",
    "startTime": 1553358512167,
    "createTime": 1553358507406,
    "context": "default",
    "params": {
      "filePath": "hello-mist-scala_0.0.1.jar",
      "className": "HelloMist$",
      "arguments": {
        "samples": 9
      },
      "action": "execute"
    },
    "endTime": 1553358513162,
    "jobResult": 2.2222222222222223,
    "status": "finished",
    "function": "hello-mist-scala",
    "jobId": "b044d08f-9554-4be9-8e22-1d687e58c52e",
    "workerId": "default_1cb3b66d-99c3-400b-ac3a-f11d72ab8124_4"
  },

(snip)

$ curl -X GET 'http://10.0.0.209:2004/v2/api/jobs' | jq

$ curl -X GET 'http://10.0.0.209:2004/v2/api/jobs/a87197c8-1692-48bc-b151-978ea89b058a/logs' | head -n 20
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0INFO 2019-03-23T16:22:44.178 [a87197c8-1692-48bc-b151-978ea89b058a] Waiting worker connectionINFO 2019-03-23T16:22:44.183 [a87197c8-1692-48bc-b151-978ea89b058a] InitializedEvent(externalId=None)
INFO 2019-03-23T16:22:44.183 [a87197c8-1692-48bc-b151-978ea89b058a] QueuedEventINFO 2019-03-23T16:22:47.991 [a87197c8-1692-48bc-b151-978ea89b058a] WorkerAssigned(workerId=default_1cb3b66d-99c3-400b-ac3a-f11d72ab8124_2)
INFO 2019-03-23T16:22:48.027 [a87197c8-1692-48bc-b151-978ea89b058a] JobFileDownloadingEvent
INFO 2019-03-23T16:22:48.885 [a87197c8-1692-48bc-b151-978ea89b058a] StartedEvent
INFO 2019-03-23T16:22:48.882 [a87197c8-1692-48bc-b151-978ea89b058a] Added JAR /home/centos/HydroMist/default/worker-default_1cb3b66d-99c3-400b-ac3a-f11d72ab8124_2/hello-mist-scala_0.0.1.jar at spark://dev:46260/jars/hello-mist-scala_0.0.1.jar with timestamp 1553358168882
INFO 2019-03-23T16:22:48.965 [a87197c8-1692-48bc-b151-978ea89b058a] Hello Mist started with samples: 8
INFO 2019-03-23T16:22:49.204 [a87197c8-1692-48bc-b151-978ea89b058a] Starting job: count at HelloMist.scala:22
INFO 2019-03-23T16:22:49.218 [a87197c8-1692-48bc-b151-978ea89b058a] Got job 0 (count at HelloMist.scala:22) with 16 output partitions
INFO 2019-03-23T16:22:49.219 [a87197c8-1692-48bc-b151-978ea89b058a] Final stage: ResultStage 0 (count at HelloMist.scala:22)
INFO 2019-03-23T16:22:49.22 [a87197c8-1692-48bc-b151-978ea89b058a] Parents of final stage: List()
INFO 2019-03-23T16:22:49.221 [a87197c8-1692-48bc-b151-978ea89b058a] Missing parents: List()
INFO 2019-03-23T16:22:49.229 [a87197c8-1692-48bc-b151-978ea89b058a] Submitting ResultStage 0 (MapPartitionsRDD[1] at filter at HelloMist.scala:18), which has
no missing parents
INFO 2019-03-23T16:22:49.438 [a87197c8-1692-48bc-b151-978ea89b058a] Block broadcast_0 stored as values in memory (estimated size 1808.0 B, free 366.3 MB)
INFO 2019-03-23T16:22:49.471 [a87197c8-1692-48bc-b151-978ea89b058a] Block broadcast_0_piece0 stored as bytes in memory (estimated size 1232.0 B, free 366.3 MB)
INFO 2019-03-23T16:22:49.473 [a87197c8-1692-48bc-b151-978ea89b058a] Added broadcast_0_piece0 in memory on dev:39976 (size: 1232.0 B, free: 366.3 MB)
INFO 2019-03-23T16:22:49.476 [a87197c8-1692-48bc-b151-978ea89b058a] Created broadcast 0 from broadcast at DAGScheduler.scala:1039
10INFO 2019-03-23T16:22:49.494 [a87197c8-1692-48bc-b151-978ea89b058a] Submitting 16 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at filter at HelloMist.scala:18) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14))
0INFO 2019-03-23T16:22:49.495 [a87197c8-1692-48bc-b151-978ea89b058a] Adding task set 0.0 with 16 tasks

(snip)

diff --git a/keyboards/crkbd/keymaps/dobachi/keymap.c b/keyboards/crkbd/keymaps/dobachi/keymap.c
index f80eff9..44a9137 100644
--- a/keyboards/crkbd/keymaps/dobachi/keymap.c
+++ b/keyboards/crkbd/keymaps/dobachi/keymap.c
@@ -158,7 +158,7 @@ void matrix_render_user(struct CharacterMatrix *matrix) {
     // If you want to change the display of OLED, you need to change here
     matrix_write_ln(matrix, read_layer_state());
     matrix_write_ln(matrix, read_keylog());
-    matrix_write_ln(matrix, read_keylogs());
+    //matrix_write_ln(matrix, read_keylogs());
     //matrix_write_ln(matrix, read_mode_icon(keymap_config.swap_lalt_lgui));
     //matrix_write_ln(matrix, read_host_led_state());
     //matrix_write_ln(matrix, read_timelog());

  private def controlledShutdown() {

    def node(broker: Broker): Node = broker.node(config.interBrokerListenerName)

(snip)

// attach shutdown handler to catch terminating signals as well as normal termination
Runtime.getRuntime().addShutdownHook(new Thread("kafka-shutdown-hook") {
  override def run(): Unit = kafkaServerStartable.shutdown()
})