Updated to 2.7.1.

- Use standard way to navigate to manage view.
- Use standard way to wait for closing modal dialog.
- Selected user is not system state agnostic.
- Take screenshot if group selection fails.

.gitlab-ci.yml

@@ -1,6 +1,23 @@
-variables:
-  Z_LOCALES: "en-us:de-de"
+image: registry.znuny.com/docker/zammad-ruby:2.4.4
 
+# Global variables added to the ENV of each job
+variables:
+  # for faster translation loading
+  Z_LOCALES: "en-us:de-de"
+  # Browser tests and variables
+  RAILS_SERVE_STATIC_FILES: "true"
+  REMOTE_URL: "http://selenium:24444/wd/hub"
+  TZ: "Europe/London"
+  # docker elgalu/selenium variables for parallel browser instance creation
+  MAX_INSTANCES: "50"
+  MAX_SESSIONS: "50"
+  # docker-imap-devel variables required for browser tests
+  MAILNAME: "mail.test.dc.zammad.com"
+  MAILBOX_INIT: "zammad@mail.test.dc.zammad.com:zammad"
+  MAIL_ADDRESS: "zammad@mail.test.dc.zammad.com"
+  MAIL_PASS: "zammad"
+
+# Artifacts are stored for failed jobs for 2 days
 .artifacts_error_template: &artifacts_error
   artifacts:
     expire_in: 2 days
@@ -9,13 +26,43 @@ variables:
     - tmp/screenshot*
     - log/*.log
 
-before_script:
-  - ruby -v
-  - which ruby
-  - env
-  - test -n "$RNAME" && script/build/test_db_config.sh
-  - test -n "$RNAME" && bundle install -j $(nproc)
+# Workaround to enable usage of mixed SSH and Docker GitLab CI runners
+.docker_env_template: &docker_env
+  tags:
+    - docker
 
+# Workaround for blocked port 25 access on cloud provider infrastructure
+.requires_mail_port_access_template: &requires_mail_port_access
+  tags:
+    - mail
+
+.base_env_template: &base_env
+  <<: *docker_env
+  <<: *artifacts_error
+
+# General required Docker services for different DB envs
+.services_mysql_template: &services_mysql
+  services:
+    - name: registry.znuny.com/docker/zammad-mysql:latest
+      alias: mysql
+
+.services_postgresql_template: &services_postgresql
+  services:
+    - name: registry.znuny.com/docker/zammad-postgresql:latest
+      alias: postgresql
+
+# Cache gems in between jobs and pipelines
+cache:
+  key: "ruby24"
+  paths:
+    - vendor/ruby
+
+# Initialize application env
+before_script:
+  - bundle install -j $(nproc) --path vendor
+  - bundle exec ruby script/build/database_config.rb
+
+# Stages
 stages:
   - pre
   - test
@@ -24,31 +71,33 @@ stages:
 
 # pre stage
 
-pre:rubocop:
+# Workaround to enable usage of mixed SSH and Docker GitLab CI runners
+.pre_stage_template: &pre_stage
+  <<: *docker_env
   stage: pre
-  tags:
-    - core
+  before_script:
+    - '' # disable before_script for pre "non-application" env
+
+pre:rubocop:
+  <<: *pre_stage
   script:
+    - bundle install -j $(nproc) --path vendor
     - bundle exec rubocop
 
 pre:coffeelint:
-  stage: pre
-  tags:
-    - core
+  <<: *pre_stage
   script:
     - coffeelint app/
 
 pre:bundle-audit:
-  stage: pre
-  tags:
-    - core
+  <<: *pre_stage
   script:
     - gem install bundler-audit
     - bundle-audit update
     - bundle-audit
 
 pre:github:
-  stage: pre
+  <<: *pre_stage
   tags:
     - deploy
   script:
@@ -59,266 +108,196 @@ pre:github:
 ## RSpec
 
 .script_rspec_template: &script_rspec_definition
-  <<: *artifacts_error
+  <<: *base_env
   variables:
     RAILS_ENV: "test"
   script:
-    - rake zammad:db:init
+    - bundle exec rake zammad:db:init
     - bundle exec rspec
-    - rake db:drop
 
 test:rspec:mysql:
   stage: test
-  tags:
-    - core
-    - mysql
+  <<: *services_mysql
   <<: *script_rspec_definition
 
 test:rspec:postgresql:
   stage: test
-  tags:
-    - core
-    - postgresql
+  <<: *services_postgresql
   <<: *script_rspec_definition
 
 ## Unit and Controller tests
 
 .script_unit_template: &script_unit_definition
-  <<: *artifacts_error
+  <<: *base_env
   variables:
     RAILS_ENV: "test"
   script:
-    - rake zammad:db:init
-    - rake test:units
-    - ruby -I test/ test/integration/object_manager_test.rb
-    - ruby -I test/ test/integration/object_manager_attributes_controller_test.rb
-    - ruby -I test/ test/integration/package_test.rb
-    - ruby -I test/ test/integration/monitoring_controller_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:init
+    - bundle exec rake test:units
+    - bundle exec ruby -I test/ test/integration/object_manager_test.rb
+    - bundle exec ruby -I test/ test/integration/object_manager_attributes_controller_test.rb
+    - bundle exec ruby -I test/ test/integration/package_test.rb
+    - bundle exec ruby -I test/ test/integration/monitoring_controller_test.rb
 
 test:unit:mysql:
   stage: test
-  tags:
-    - core
-    - mysql
+  <<: *services_mysql
   <<: *script_unit_definition
 
 test:unit:postgresql:
   stage: test
-  tags:
-    - core
-    - postgresql
+  <<: *services_postgresql
   <<: *script_unit_definition
 
 ## Integration tests
 
-test:integration:email_helper_deliver:
-  <<: *artifacts_error
+.test_integration_template: &test_integration_definition
+  <<: *base_env
+  <<: *services_postgresql
   stage: test
   variables:
     RAILS_ENV: "test"
-  tags:
-    - core
+
+test:integration:email_helper_deliver:
+  <<: *test_integration_definition
+  <<: *requires_mail_port_access
   script:
-    - rake zammad:db:unseeded
-    - ruby -I test/ test/integration/email_helper_test.rb
-    - ruby -I test/ test/integration/email_deliver_test.rb
-    - ruby -I test/ test/integration/email_keep_on_server_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:unseeded
+    - bundle exec ruby -I test/ test/integration/email_helper_test.rb
+    - bundle exec ruby -I test/ test/integration/email_deliver_test.rb
+    - bundle exec ruby -I test/ test/integration/email_keep_on_server_test.rb
 
 test:integration:twitter:
-  <<: *artifacts_error
-  stage: test
-  variables:
-    RAILS_ENV: "test"
-  tags:
-    - core-twitter
+  <<: *test_integration_definition
   script:
-    - rake zammad:db:init
-    - ruby -I test/ test/integration/twitter_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:init
+    - bundle exec ruby -I test/ test/integration/twitter_test.rb
   allow_failure: true
 
 test:integration:facebook:
-  <<: *artifacts_error
-  stage: test
-  variables:
-    RAILS_ENV: "test"
-  tags:
-    - core-facebook
+  <<: *test_integration_definition
   script:
-    - rake zammad:db:init
-    - ruby -I test/ test/integration/facebook_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:init
+    - bundle exec ruby -I test/ test/integration/facebook_test.rb
   allow_failure: true
 
 test:integration:geo:
-  <<: *artifacts_error
-  stage: test
-  variables:
-    RAILS_ENV: "test"
-  tags:
-    - core
+  <<: *test_integration_definition
   script:
-    - rake zammad:db:unseeded
-    - ruby -I test/ test/integration/geo_calendar_test.rb
-    - ruby -I test/ test/integration/geo_location_test.rb
-    - ruby -I test/ test/integration/geo_ip_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:unseeded
+    - bundle exec ruby -I test/ test/integration/geo_calendar_test.rb
+    - bundle exec ruby -I test/ test/integration/geo_location_test.rb
+    - bundle exec ruby -I test/ test/integration/geo_ip_test.rb
 
 test:integration:user_agent:
-  <<: *artifacts_error
-  stage: test
-  variables:
-    RAILS_ENV: "test"
-  tags:
-    - core
+  <<: *test_integration_definition
   script:
-    - rake zammad:db:unseeded
-    - ruby -I test/ test/integration/user_agent_test.rb
+    - bundle exec rake zammad:db:unseeded
+    - bundle exec ruby -I test/ test/integration/user_agent_test.rb
     - export ZAMMAD_PROXY_TEST=true
-    - ruby -I test/ test/integration/user_agent_test.rb
-    - rake db:drop
+    - bundle exec ruby -I test/ test/integration/user_agent_test.rb
   allow_failure: true
 
 test:integration:user_device:
-  <<: *artifacts_error
-  stage: test
-  variables:
-    RAILS_ENV: "test"
-  tags:
-    - core
+  <<: *test_integration_definition
   script:
-    - rake zammad:db:unseeded
-    - ruby -I test/ test/integration/user_device_controller_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:unseeded
+    - bundle exec ruby -I test/ test/integration/user_device_controller_test.rb
 
 test:integration:slack:
-  <<: *artifacts_error
-  stage: test
-  variables:
-    RAILS_ENV: "test"
-  tags:
-    - core
+  <<: *test_integration_definition
   script:
-    - rake zammad:db:unseeded
+    - bundle exec rake zammad:db:unseeded
     - echo "gem 'slack-api'" >> Gemfile.local
     - bundle install -j $(nproc)
-    - ruby -I test test/integration/slack_test.rb
-    - rake db:drop
+    - bundle exec ruby -I test test/integration/slack_test.rb
 
 test:integration:clearbit:
-  <<: *artifacts_error
-  stage: test
-  variables:
-    RAILS_ENV: "test"
-  tags:
-    - core
+  <<: *test_integration_definition
   script:
-    - rake zammad:db:unseeded
-    - ruby -I test test/integration/clearbit_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:unseeded
+    - bundle exec ruby -I test test/integration/clearbit_test.rb
   allow_failure: true
 
 test:integration:telegram:
-  <<: *artifacts_error
-  stage: test
-  variables:
-    RAILS_ENV: "test"
-  tags:
-    - core
+  <<: *test_integration_definition
   script:
-    - rake zammad:db:init
-    - ruby -I test test/integration/telegram_controller_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:init
+    - bundle exec ruby -I test test/integration/telegram_controller_test.rb
 
 test:integration:twilio:
-  <<: *artifacts_error
-  stage: test
-  variables:
-    RAILS_ENV: "test"
-  tags:
-    - core
+  <<: *test_integration_definition
   script:
-    - rake zammad:db:init
-    - ruby -I test test/integration/twilio_sms_controller_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:init
+    - bundle exec ruby -I test test/integration/twilio_sms_controller_test.rb
 
 test:integration:idoit:
-  <<: *artifacts_error
-  stage: test
-  variables:
-    RAILS_ENV: "test"
-  tags:
-    - core
+  <<: *test_integration_definition
   script:
-    - rake zammad:db:init
-    - ruby -I test test/integration/idoit_controller_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:init
+    - bundle exec ruby -I test test/integration/idoit_controller_test.rb
 
 ### Elasticsearch
 
 .script_elasticsearch_template: &script_elasticsearch_definition
-  <<: *artifacts_error
+  <<: *base_env
   stage: test
   variables:
     RAILS_ENV: "test"
     ES_INDEX_RAND: "true"
-    ES_URL: "http://localhost:9200"
+    ES_URL: "http://elasticsearch:9200"
   script:
-    - rake zammad:db:unseeded
-    - ruby -I test/ test/integration/elasticsearch_active_test.rb
-    - ruby -I test/ test/integration/elasticsearch_test.rb
-    - ruby -I test/ test/integration/report_test.rb
+    - bundle exec rake zammad:db:unseeded
+    - bundle exec ruby -I test/ test/integration/elasticsearch_active_test.rb
+    - bundle exec ruby -I test/ test/integration/elasticsearch_test.rb
+    - bundle exec ruby -I test/ test/integration/report_test.rb
     - bundle exec rspec --tag searchindex
-    - rake db:drop
 
 test:integration:es_mysql:
-  tags:
-    - core
-    - mysql
   <<: *script_elasticsearch_definition
+  services:
+    - name: registry.znuny.com/docker/zammad-mysql:latest
+      alias: mysql
+    - name: registry.znuny.com/docker/zammad-elasticsearch:latest
+      alias: elasticsearch
 
 test:integration:es_postgresql:
-  tags:
-    - core
-    - postgresql
   <<: *script_elasticsearch_definition
+  services:
+    - name: registry.znuny.com/docker/zammad-postgresql:latest
+      alias: postgresql
+    - name: registry.znuny.com/docker/zammad-elasticsearch:latest
+      alias: elasticsearch
 
 ### Zendesk
 
 .script_integration_zendesk_template: &script_integration_zendesk_definition
-  <<: *artifacts_error
+  <<: *base_env
+  <<: *services_postgresql
   stage: test
   variables:
     RAILS_ENV: "test"
   script:
-    - rake zammad:db:unseeded
-    - ruby -I test/ test/integration/zendesk_import_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:unseeded
+    - bundle exec ruby -I test/ test/integration/zendesk_import_test.rb
   allow_failure: true
 
 test:integration:zendesk_mysql:
-  tags:
-    - core
-    - mysql
+  <<: *services_mysql
   <<: *script_integration_zendesk_definition
 
 test:integration:zendesk_postgresql:
-  tags:
-    - core
-    - postgresql
+  <<: *services_postgresql
   <<: *script_integration_zendesk_definition
 
 ### OTRS
 
 .script_integration_otrs_template: &script_integration_otrs_definition
-  <<: *artifacts_error
+  <<: *base_env
   stage: test
   script:
-    - rake zammad:db:unseeded
-    - ruby -I test/ test/integration/otrs_import_test.rb
-    - rake db:drop
+    - bundle exec rake zammad:db:unseeded
+    - bundle exec ruby -I test/ test/integration/otrs_import_test.rb
 
 .variables_integration_otrs_6_template: &variables_integration_otrs_6_definition
   variables:
@@ -326,54 +305,45 @@ test:integration:zendesk_postgresql:
     IMPORT_OTRS_ENDPOINT: "https://vz1185.test.znuny.com/otrs/public.pl?Action=ZammadMigrator"
 
 test:integration:otrs_6_mysql:
-  tags:
-    - core
-    - mysql
+  <<: *services_mysql
   <<: *script_integration_otrs_definition
   <<: *variables_integration_otrs_6_definition
 
 test:integration:otrs_6_postgresql:
-  tags:
-    - core
-    - postgresql
+  <<: *services_postgresql
   <<: *script_integration_otrs_definition
   <<: *variables_integration_otrs_6_definition
 
 test:integration:otrs_5:
-  tags:
-    - core
+  <<: *services_postgresql
   variables:
     RAILS_ENV: "test"
     IMPORT_OTRS_ENDPOINT: "http://vz1109.demo.znuny.com/otrs/public.pl?Action=ZammadMigrator"
   <<: *script_integration_otrs_definition
 
 test:integration:otrs_4:
-  tags:
-    - core
+  <<: *services_postgresql
   variables:
     RAILS_ENV: "test"
     IMPORT_OTRS_ENDPOINT: "http://vz383.demo.znuny.com/otrs/public.pl?Action=ZammadMigrator"
   <<: *script_integration_otrs_definition
 
 test:integration:otrs_33:
-  tags:
-    - core
+  <<: *services_postgresql
   variables:
     RAILS_ENV: "test"
     IMPORT_OTRS_ENDPOINT: "http://vz305.demo.znuny.com/otrs/public.pl?Action=ZammadMigrator"
   <<: *script_integration_otrs_definition
 
 test:integration:otrs_32:
-  tags:
-    - core
+  <<: *services_postgresql
   variables:
     RAILS_ENV: "test"
     IMPORT_OTRS_ENDPOINT: "http://vz382.demo.znuny.com/otrs/public.pl?Action=ZammadMigrator"
   <<: *script_integration_otrs_definition
 
 test:integration:otrs_31:
-  tags:
-    - core
+  <<: *services_postgresql
   variables:
     RAILS_ENV: "test"
     IMPORT_OTRS_ENDPOINT: "http://vz381.demo.znuny.com/otrs/public.pl?Action=ZammadMigrator"
@@ -384,15 +354,14 @@ test:integration:otrs_31:
 ## preparation (asset precompile)
 
 browser:build:
+  <<: *base_env
+  <<: *services_postgresql
   stage: test
-  tags:
-    - core
   variables:
     RAILS_ENV: "production"
   script:
-    - rake zammad:db:unseeded
-    - rake assets:precompile
-    - rake db:drop
+    - bundle exec rake zammad:db:unseeded
+    - bundle exec rake assets:precompile
   artifacts:
     expire_in: 1 week
     paths:
@@ -403,13 +372,11 @@ browser:build:
 ## Browser core tests
 
 .variables_browser_template: &variables_browser_definition
-  RAILS_SERVE_STATIC_FILES: "true"
   RAILS_ENV: "production"
-  BROWSER_URL: "http://$IP:$BROWSER_PORT"
-  APP_RESTART_CMD: "rake zammad:ci:app:restart"
+  APP_RESTART_CMD: "bundle exec rake zammad:ci:app:restart"
 
 .test_browser_core_template: &test_browser_core_definition
-  <<: *artifacts_error
+  <<: *base_env
   stage: browser-core
   dependencies:
     - browser:build
@@ -418,47 +385,42 @@ browser:build:
 
 test:browser:integration:api_client_ruby:
   <<: *test_browser_core_definition
+  <<: *services_postgresql
   variables:
     <<: *variables_browser_definition
-    TEST_URL: "http://$IP:$BROWSER_PORT/"
-  tags:
-    - browser
   script:
-    - RAILS_ENV=test rake db:create
+    - RAILS_ENV=test bundle exec rake db:create
     - cp contrib/auto_wizard_test.json auto_wizard.json
-    - rake zammad:ci:test:start
-    - git clone git@github.com:zammad/zammad-api-client-ruby.git || rake zammad:ci:test:fail
+    - bundle exec rake zammad:ci:test:start
+    - git clone https://github.com/zammad/zammad-api-client-ruby.git
     - cd zammad-api-client-ruby
     - bundle install -j $(nproc)
-    - bundle exec rspec || (cd .. && rake zammad:ci:test:fail)
-    - cd .. && rake zammad:ci:test:stop
+    - bundle exec rspec
 
 test:browser:integration:api_client_php:
   <<: *test_browser_core_definition
+  <<: *services_postgresql
   variables:
     <<: *variables_browser_definition
-    ZAMMAD_PHP_API_CLIENT_UNIT_TESTS_URL: "http://$IP:$BROWSER_PORT"
+    ZAMMAD_PHP_API_CLIENT_UNIT_TESTS_URL: "http://localhost:3000"
     ZAMMAD_PHP_API_CLIENT_UNIT_TESTS_USERNAME: "master@example.com"
     ZAMMAD_PHP_API_CLIENT_UNIT_TESTS_PASSWORD: "test"
-  tags:
-    - browser
   script:
-    - RAILS_ENV=test rake db:create
-    - rake zammad:ci:test:start zammad:setup:auto_wizard
-    - git clone git@github.com:zammad/zammad-api-client-php || rake zammad:ci:test:fail
+    - RAILS_ENV=test bundle exec rake db:create
+    - bundle exec rake zammad:ci:test:start zammad:setup:auto_wizard
+    - git clone https://github.com/zammad/zammad-api-client-php.git
+    - php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
+    - php composer-setup.php --install-dir=/usr/local/bin
+    - ln -s /usr/local/bin/composer.phar /usr/local/bin/composer
     - cd zammad-api-client-php
     - composer install
-    - vendor/bin/phpunit || (cd .. && rake zammad:ci:test:fail)
-    - cd .. && rake zammad:ci:test:stop
+    - vendor/bin/phpunit
 
 ### Browser test slices
 
 #### Templates
 
 .script_browser_slice_template: &script_browser_slice_definition
-  stage: browser-core
-  dependencies:
-    - browser:build
   script:
     # temporary workaround to check Yahoo! mailbox only in test:browser:core:ff_3_* tests
     - if [[ $CI_JOB_NAME != test:browser:core:ff_3_* ]]; then unset MAILBOX_AUTO1 ; fi
@@ -467,24 +429,35 @@ test:browser:integration:api_client_php:
     - if [[ $CI_JOB_NAME != test:browser:core:ff_3_* ]]; then unset MAILBOX_MANUAL2 ; fi
     - env
     - script/build/test_slice_tests.sh $TEST_SLICE
-    - RAILS_ENV=test rake db:create
-    - rake zammad:ci:test:start[with_elasticsearch]
-    - rake test:browser || rake zammad:ci:test:fail
-    - rake zammad:ci:test:stop
+    - RAILS_ENV=test bundle exec rake db:create
+    - bundle exec rake zammad:ci:test:start[with_elasticsearch]
+    - bundle exec rake test:browser
 
 .test_browser_core_postgresql_template: &test_browser_core_postgresql_definition
   <<: *test_browser_core_definition
-  tags:
-    - browser
-    - postgresql
   <<: *script_browser_slice_definition
+  services:
+    - name: registry.znuny.com/docker/zammad-postgresql:latest
+      alias: postgresql
+    - name: registry.znuny.com/docker/zammad-elasticsearch:latest
+      alias: elasticsearch
+    - name: docker.io/elgalu/selenium:latest
+      alias: selenium
+    - name: registry.znuny.com/docker/docker-imap-devel:latest
+      alias: mail
 
 .test_browser_core_mysql_template: &test_browser_core_mysql_definition
   <<: *test_browser_core_definition
-  tags:
-    - browser
-    - mysql
   <<: *script_browser_slice_definition
+  services:
+    - name: registry.znuny.com/docker/zammad-mysql:latest
+      alias: mysql
+    - name: registry.znuny.com/docker/zammad-elasticsearch:latest
+      alias: elasticsearch
+    - name: docker.io/elgalu/selenium:latest
+      alias: selenium
+    - name: registry.znuny.com/docker/docker-imap-devel:latest
+      alias: mail
 
 #### Firefox
 
@@ -504,6 +477,7 @@ test:browser:core:ff_2_mysql:
 
 test:browser:core:ff_3_mysql:
   <<: *test_browser_core_mysql_definition
+  <<: *requires_mail_port_access
   variables:
     <<: *variables_browser_definition
     BROWSER: "firefox"
@@ -546,6 +520,7 @@ test:browser:core:ff_2_postgresql:
 
 test:browser:core:ff_3_postgresql:
   <<: *test_browser_core_postgresql_definition
+  <<: *requires_mail_port_access
   variables:
     <<: *variables_browser_definition
     BROWSER: "firefox"
@@ -660,28 +635,34 @@ test:browser:core:chrome_6_postgresql:
 
 ### Auto wizard
 
+.auto_wizard_services_template: &auto_wizard_services
+  services:
+    - name: registry.znuny.com/docker/zammad-postgresql:latest
+      alias: postgresql
+    - name: docker.io/elgalu/selenium:latest
+      alias: selenium
+
 .test_browser_integration_template: &test_browser_integration_definition
-  <<: *artifacts_error
+  <<: *base_env
+  <<: *auto_wizard_services
   stage: browser-integration
   dependencies:
     - browser:build
 
 .script_integration_auto_wizard_template: &script_integration_auto_wizard_definition
   script:
-    - RAILS_ENV=test rake db:create
+    - RAILS_ENV=test bundle exec rake db:create
     - cp $AUTO_WIZARD_FILE auto_wizard.json
-    - rake zammad:ci:test:start[with_elasticsearch]
-    - ruby -I test/ $TEST_FILE || rake zammad:ci:test:fail
-    - rake zammad:ci:test:stop
+    - bundle exec rake zammad:ci:test:start
+    - bundle exec ruby -I test/ $TEST_FILE
 
 .browser_core_auto_wizard_template: &browser_core_auto_wizard_definition
   <<: *test_browser_core_definition
+  <<: *auto_wizard_services
   <<: *script_integration_auto_wizard_definition
 
 test:browser:autowizard_chrome:
   <<: *browser_core_auto_wizard_definition
-  tags:
-    - browser
   variables:
     <<: *variables_browser_definition
     BROWSER: "chrome"
@@ -690,8 +671,6 @@ test:browser:autowizard_chrome:
 
 test:browser:autowizard_ff:
   <<: *browser_core_auto_wizard_definition
-  tags:
-    - browser
   variables:
     <<: *variables_browser_definition
     BROWSER: "firefox"
@@ -706,9 +685,6 @@ test:browser:autowizard_ff:
 
 test:browser:integration:twitter_chrome:
   <<: *browser_integration_auto_wizard_definition
-  tags:
-    - browser
-    - twitter
   variables:
     <<: *variables_browser_definition
     BROWSER: "chrome"
@@ -717,9 +693,6 @@ test:browser:integration:twitter_chrome:
 
 test:browser:integration:twitter_ff:
   <<: *browser_integration_auto_wizard_definition
-  tags:
-    - browser
-    - twitter
   variables:
     <<: *variables_browser_definition
     BROWSER: "firefox"
@@ -728,9 +701,6 @@ test:browser:integration:twitter_ff:
 
 test:browser:integration:facebook_chrome:
   <<: *browser_integration_auto_wizard_definition
-  tags:
-    - browser
-    - facebook
   variables:
     <<: *variables_browser_definition
     BROWSER: "chrome"
@@ -739,9 +709,6 @@ test:browser:integration:facebook_chrome:
 
 test:browser:integration:facebook_ff:
   <<: *browser_integration_auto_wizard_definition
-  tags:
-    - browser
-    - facebook
   variables:
     <<: *variables_browser_definition
     BROWSER: "firefox"
@@ -750,8 +717,6 @@ test:browser:integration:facebook_ff:
 
 test:browser:integration:idoit_chrome:
   <<: *browser_integration_auto_wizard_definition
-  tags:
-    - browser
   variables:
     <<: *variables_browser_definition
     BROWSER: "chrome"
@@ -762,17 +727,15 @@ test:browser:integration:idoit_chrome:
 
 .variables_browser_import_template: &variables_browser_import_definition
   BROWSER: "chrome"
-  BROWSER_URL: "http://$IP:$BROWSER_PORT"
+  RAILS_SERVE_STATIC_FILES: "true"
+  RAILS_ENV: "production"
 
 .browser_integration_import_template: &browser_integration_import_definition
   <<: *test_browser_integration_definition
-  tags:
-    - browser
   script:
-    - RAILS_ENV=test rake db:create
-    - rake zammad:ci:test:start
-    - ruby -I test/ $TEST_FILE
-    - rake zammad:ci:test:stop
+    - RAILS_ENV=test bundle exec rake db:create
+    - bundle exec rake zammad:ci:test:start
+    - bundle exec ruby -I test/ $TEST_FILE
 
 test:browser:integration:otrs_chrome:
   <<: *browser_integration_import_definition

.rubocop.yml

@@ -10,6 +10,7 @@ AllCops:
     - 'bin/rake'
     - 'bin/spring'
     - 'db/schema.rb'
+    - 'vendor/**/*'
 
 Rails:
   Enabled: true

CHANGELOG.md

@@ -1,16 +1,101 @@
 # Change Log
 
-## [2.7.0](https://github.com/zammad/zammad/tree/2.7.0) (2018-xx-xx)
+## [2.7.2](https://github.com/zammad/zammad/tree/2.7.2) (2019-02-14)
+[Full Changelog](https://github.com/zammad/zammad/compare/2.7.1...2.7.2)
+
+**Fixed bugs:**
+- Improved header handling.
+
+
+## [2.7.1](https://github.com/zammad/zammad/tree/2.7.1) (2018-12-03)
+[Full Changelog](https://github.com/zammad/zammad/compare/2.7.0...2.7.1)
+
+**Implemented enhancements:**
+- Logs flooded with article autosave data [2363](https://github.com/zammad/zammad/issues/2363) [[enhancement](https://github.com/zammad/zammad/labels/enhancement)]
+
+**Fixed bugs:**
+- Unable to process html email with 2k links in an acceptable time [2374](https://github.com/zammad/zammad/issues/2374) [[bug](https://github.com/zammad/zammad/labels/bug)] [[needs verification](https://github.com/zammad/zammad/labels/needs verification)]
+- Zendesk-Import fails for User & Organizations if Plan is below "Team" [2262](https://github.com/zammad/zammad/issues/2262) [[bug](https://github.com/zammad/zammad/labels/bug)] [[import](https://github.com/zammad/zammad/labels/import)]
+- Chat does not work due to errors in WebSocket (Database connection) [2353](https://github.com/zammad/zammad/issues/2353) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- selecting the overviews on the I Pad does not work [2330](https://github.com/zammad/zammad/issues/2330) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Lost DB connection causes jobs to not be processed anymore [2343](https://github.com/zammad/zammad/issues/2343) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Role-Filter shows inactive Roles [2332](https://github.com/zammad/zammad/issues/2332) [[admin area](https://github.com/zammad/zammad/labels/admin area)] [[bug](https://github.com/zammad/zammad/labels/bug)]
+- IMAP fetch stops because of request timeout [2321](https://github.com/zammad/zammad/issues/2321) [[bug](https://github.com/zammad/zammad/labels/bug)] [[channel](https://github.com/zammad/zammad/labels/channel)]
+- Unable to import (update) users with email addresses als lookup index written in upper letters [2329](https://github.com/zammad/zammad/issues/2329) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Websocket messages are not working correctly via Ajax long polling (e. g. multiple browser tabs can get opened with one session). [2327](https://github.com/zammad/zammad/issues/2327) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Improved error handling of invalid tag for text modules and add current time format (to not use 2018-10-31T08:02:21.917Z format) [2324](https://github.com/zammad/zammad/issues/2324) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Ticket list in overview of customer and organisation random order [2296](https://github.com/zammad/zammad/issues/2296) [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)]
+- Can't forward E-Mail in special cases within Firefox [2305](https://github.com/zammad/zammad/issues/2305) [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)]
+
+
+## [2.7.0](https://github.com/zammad/zammad/tree/2.7.0) (2018-10-25)
 [Full Changelog](https://github.com/zammad/zammad/compare/2.6.0...2.7.0)
 
 **Implemented enhancements:**
-
-
-
+- O365/Office365 authentication missing given- and surname [2281](https://github.com/zammad/zammad/issues/2281) [[enhancement](https://github.com/zammad/zammad/labels/enhancement)]
+- Callback issue when Zammad if internal and external FQDN differ [2223](https://github.com/zammad/zammad/issues/2223) [[CTI](https://github.com/zammad/zammad/labels/CTI)] [[admin area](https://github.com/zammad/zammad/labels/admin area)] [[enhancement](https://github.com/zammad/zammad/labels/enhancement)] [[frontend](https://github.com/zammad/zammad/labels/frontend)]
+- Italian and Spanish translations for notifications. [2270](https://github.com/zammad/zammad/issues/2270) [[enhancement](https://github.com/zammad/zammad/labels/enhancement)]
+- Make stats store and cti log searchable via elastichsearch & destroy stats store if user got deleted [2261](https://github.com/zammad/zammad/issues/2261) [[enhancement](https://github.com/zammad/zammad/labels/enhancement)]
+- Global Bcc for all outgoing emails for external archive options. [2228](https://github.com/zammad/zammad/issues/2228) [[enhancement](https://github.com/zammad/zammad/labels/enhancement)]
+- Added custom notification templates to make changes update save [2081](https://github.com/zammad/zammad/pull/2081) [[enhancement](https://github.com/zammad/zammad/labels/enhancement)] [[notification](https://github.com/zammad/zammad/labels/notification)]
+- WebApp (monitoring section) claims scheduler not running, if delayed jobs get too many [2188](https://github.com/zammad/zammad/issues/2188) [[enhancement](https://github.com/zammad/zammad/labels/enhancement)]
 
 **Fixed bugs:**
-
-
-
-
-\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*
+- Reindex elastic search not possible because of <null>/Twitter::NullObject [1394](https://github.com/zammad/zammad/issues/1394) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- `rake searchindex:rebuild` is not working after Elasticsearch 2.4 upgrade to Elasticsearch 5.6 "Limit of total fields [1000] in index" [2297](https://github.com/zammad/zammad/issues/2297) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Update bot_add.jst.eco [2295](https://github.com/zammad/zammad/pull/2295)
+- CTI Log: duration_talking_time is incorrect and after update initialized_at is a string (not timestamp) [2292](https://github.com/zammad/zammad/issues/2292) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Performance issue in bulk action selection in overview [2279](https://github.com/zammad/zammad/issues/2279) [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)] [[overviews](https://github.com/zammad/zammad/labels/overviews)] [[regression](https://github.com/zammad/zammad/labels/regression)]
+- Note is not shown for customer / organisations if it's empty. [2277](https://github.com/zammad/zammad/issues/2277) [[bug](https://github.com/zammad/zammad/labels/bug)] [[regression](https://github.com/zammad/zammad/labels/regression)]
+- Permission issue: response templates only usable when admin is also agent [2285](https://github.com/zammad/zammad/issues/2285) [[admin area](https://github.com/zammad/zammad/labels/admin area)] [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Bulk Form javascript error when checking item [2273](https://github.com/zammad/zammad/issues/2273) [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)] [[needs verification](https://github.com/zammad/zammad/labels/needs verification)] [[overviews](https://github.com/zammad/zammad/labels/overviews)] [[regression](https://github.com/zammad/zammad/labels/regression)]
+- Records in Reporting not updated when single ActiveRecord can not be found [2246](https://github.com/zammad/zammad/issues/2246) [[bug](https://github.com/zammad/zammad/labels/bug)] [[reporting](https://github.com/zammad/zammad/labels/reporting)]
+- Users deactivated by the LDAP sync get updated on every run [2256](https://github.com/zammad/zammad/issues/2256) [[LDAP](https://github.com/zammad/zammad/labels/LDAP)] [[bug](https://github.com/zammad/zammad/labels/bug)] [[integration](https://github.com/zammad/zammad/labels/integration)]
+- Search with id:123 does not work anymore [2195](https://github.com/zammad/zammad/issues/2195) [[bug](https://github.com/zammad/zammad/labels/bug)] [[search](https://github.com/zammad/zammad/labels/search)]
+- Update zammad_ssl.conf [2269](https://github.com/zammad/zammad/pull/2269)
+- Fix failing Travis CI build [2268](https://github.com/zammad/zammad/pull/2268)
+- Every session comes from 127.0.0.1 [742](https://github.com/zammad/zammad/issues/742) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Reply does not work as expected [2184](https://github.com/zammad/zammad/issues/2184) [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)]
+- Invalid date causes errors [2173](https://github.com/zammad/zammad/issues/2173) [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)]
+- Check the correct hash array if the agent is present [2243](https://github.com/zammad/zammad/pull/2243) [[bug](https://github.com/zammad/zammad/labels/bug)] [[needs verification](https://github.com/zammad/zammad/labels/needs verification)]
+- CTI: Limit used caller log states and prevent race conditions (if callback comes later) [2255](https://github.com/zammad/zammad/issues/2255) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Boolean object set to false is not visible [2233](https://github.com/zammad/zammad/issues/2233) [[bug](https://github.com/zammad/zammad/labels/bug)] [[object manager attribute](https://github.com/zammad/zammad/labels/object manager attribute)]
+- Unable to process spam email `"ERROR: #<Exceptions::UnprocessableEntity: Invalid email>"` [2254](https://github.com/zammad/zammad/issues/2254) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Unable to index elasticsearch if article.preferences[:delivery_status_message] has no uf8 charset [2253](https://github.com/zammad/zammad/issues/2253) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Unable to process email `"ERROR: #<NoMethodError: undefined method `match' for nil:NilClass>"` [2252](https://github.com/zammad/zammad/issues/2252) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Fixed typo in 404: Ressource -> resource [2251](https://github.com/zammad/zammad/pull/2251)
+- Unable to process spam mail with invalid date field [2245](https://github.com/zammad/zammad/issues/2245) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Bulk action should not be shown if user has no change permissions [2026](https://github.com/zammad/zammad/issues/2026) [[bug](https://github.com/zammad/zammad/labels/bug)] [[bulk](https://github.com/zammad/zammad/labels/bulk)] [[frontend](https://github.com/zammad/zammad/labels/frontend)] [[overviews](https://github.com/zammad/zammad/labels/overviews)]
+- Naming an attribute "attribute" causes ActiveRecord failure [2236](https://github.com/zammad/zammad/issues/2236) [[bug](https://github.com/zammad/zammad/labels/bug)] [[object manager attribute](https://github.com/zammad/zammad/labels/object manager attribute)]
+- Overview setting isn't applied on submit [2235](https://github.com/zammad/zammad/issues/2235) [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)]
+- Also not active report profiles are shown in #report screen [2232](https://github.com/zammad/zammad/issues/2232) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- LDAP/Exchange UTF-8 Status Code 500 [2140](https://github.com/zammad/zammad/issues/2140) [[LDAP](https://github.com/zammad/zammad/labels/LDAP)] [[bug](https://github.com/zammad/zammad/labels/bug)] [[import](https://github.com/zammad/zammad/labels/import)]
+- In overviews conditions the tag name (not the tag id) is stored. Renaming tags will break overviews condition [1690](https://github.com/zammad/zammad/issues/1690) [[bug](https://github.com/zammad/zammad/labels/bug)] [[overviews](https://github.com/zammad/zammad/labels/overviews)]
+- Wrong Recent viewed tickets list [2194](https://github.com/zammad/zammad/issues/2194) [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)]
+- Unprocessable email ERROR: #<ArgumentError: invalid byte sequence in UTF-8> [2227](https://github.com/zammad/zammad/issues/2227) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Unable to process email (Encoding::ConverterNotFoundError: code converter not found (Windows-1258 to UTF-8)) [2224](https://github.com/zammad/zammad/issues/2224) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Printed documents do not contain the whole ticket content, if it gets too much [2162](https://github.com/zammad/zammad/issues/2162) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Email IMAP fetch stops working if mailserver is once unreachable [1861](https://github.com/zammad/zammad/issues/1861) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Unable to use exchange sync (stack level too deep (SystemStackError)) [2220](https://github.com/zammad/zammad/issues/2220) [[bug](https://github.com/zammad/zammad/labels/bug)] [[integration](https://github.com/zammad/zammad/labels/integration)]
+- Exchange import fails if folder names can't be converted to Unicode [2152](https://github.com/zammad/zammad/issues/2152) [[bug](https://github.com/zammad/zammad/labels/bug)] [[import](https://github.com/zammad/zammad/labels/import)] [[waiting for feedback](https://github.com/zammad/zammad/labels/waiting for feedback)]
+- CTI Caller Log blocks user deletion/destroy and fails [2218](https://github.com/zammad/zammad/issues/2218) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Can't process email with space in email [2198](https://github.com/zammad/zammad/issues/2198) [[bug](https://github.com/zammad/zammad/labels/bug)] [[mail processing](https://github.com/zammad/zammad/labels/mail processing)]
+- Unprocessable Mail if sender contains http.net@ [2199](https://github.com/zammad/zammad/issues/2199) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Call log shows only orga infos on event, not on site load [2075](https://github.com/zammad/zammad/issues/2075) [[CTI](https://github.com/zammad/zammad/labels/CTI)] [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)] [[needs verification](https://github.com/zammad/zammad/labels/needs verification)]
+- Chat-Widget disappears after close [2197](https://github.com/zammad/zammad/issues/2197) [[bug](https://github.com/zammad/zammad/labels/bug)] [[chat](https://github.com/zammad/zammad/labels/chat)]
+- work around for outlook feature that breaks email address standard [2154](https://github.com/zammad/zammad/issues/2154) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Upgrade from 2.0 to 2.5 db migrations canceled [2159](https://github.com/zammad/zammad/issues/2159) [[bug](https://github.com/zammad/zammad/labels/bug)] [[object manager attribute](https://github.com/zammad/zammad/labels/object manager attribute)] [[update](https://github.com/zammad/zammad/labels/update)] [[waiting for feedback](https://github.com/zammad/zammad/labels/waiting for feedback)]
+- Order of relation select fields (Group, State etc.) values is broken [2209](https://github.com/zammad/zammad/issues/2209) [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)]
+- Unable to modify tree_select attributes with fresh 2.6 [2206](https://github.com/zammad/zammad/issues/2206) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Max size of tabs is not recognised [2204](https://github.com/zammad/zammad/issues/2204) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Unable to process email `EncodingError: could not find a valid input encoding` [2200](https://github.com/zammad/zammad/issues/2200) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Email - full quote time specification [1426](https://github.com/zammad/zammad/issues/1426) [[feature backlog](https://github.com/zammad/zammad/labels/feature backlog)] [[ticket](https://github.com/zammad/zammad/labels/ticket)]
+- Bulk-Action: Not possible to change owner [1864](https://github.com/zammad/zammad/issues/1864) [[UX/UI](https://github.com/zammad/zammad/labels/UX/UI)] [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)]
+- Incorrect reset after deleting single object-selection [2020](https://github.com/zammad/zammad/issues/2020) [[bug](https://github.com/zammad/zammad/labels/bug)] [[object manager attribute](https://github.com/zammad/zammad/labels/object manager attribute)]
+- Puma performance is going worst if users with phone attribute are updated or new tickets are created. [2193](https://github.com/zammad/zammad/issues/2193) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Sometimes automatisation is not executed [2191](https://github.com/zammad/zammad/issues/2191) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Problem while updating to 2.6: Mysql2::Error: Invalid default value for 'start_at' [2183](https://github.com/zammad/zammad/issues/2183) [[bug](https://github.com/zammad/zammad/labels/bug)] [[update](https://github.com/zammad/zammad/labels/update)]
+- LDAP / Exchange update (save!) records unnecessarily [2187](https://github.com/zammad/zammad/issues/2187) [[bug](https://github.com/zammad/zammad/labels/bug)] [[import](https://github.com/zammad/zammad/labels/import)]
+- Overview not showing unassigned tickets "if not defined" (e. g. for owner but also for select or input fields) [2171](https://github.com/zammad/zammad/issues/2171) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Incorrect encoding of email messages in HTML format (Windows-1250) [2167](https://github.com/zammad/zammad/issues/2167) [[bug](https://github.com/zammad/zammad/labels/bug)]
+- Call-Log shows inactive users [2096](https://github.com/zammad/zammad/issues/2096) [[CTI](https://github.com/zammad/zammad/labels/CTI)] [[bug](https://github.com/zammad/zammad/labels/bug)] [[frontend](https://github.com/zammad/zammad/labels/frontend)]

Gemfile

@@ -2,7 +2,7 @@ source 'https://rubygems.org'
 
 # core - base
 ruby '2.4.4'
-gem 'rails', '5.1.5'
+gem 'rails', '5.1.6.1'
 
 # core - rails additions
 gem 'activerecord-import'

Gemfile.lock

@@ -23,37 +23,37 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (5.1.5)
-      actionpack (= 5.1.5)
+    actioncable (5.1.6.1)
+      actionpack (= 5.1.6.1)
       nio4r (~> 2.0)
       websocket-driver (~> 0.6.1)
-    actionmailer (5.1.5)
-      actionpack (= 5.1.5)
-      actionview (= 5.1.5)
-      activejob (= 5.1.5)
+    actionmailer (5.1.6.1)
+      actionpack (= 5.1.6.1)
+      actionview (= 5.1.6.1)
+      activejob (= 5.1.6.1)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.1.5)
-      actionview (= 5.1.5)
-      activesupport (= 5.1.5)
+    actionpack (5.1.6.1)
+      actionview (= 5.1.6.1)
+      activesupport (= 5.1.6.1)
       rack (~> 2.0)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.1.5)
-      activesupport (= 5.1.5)
+    actionview (5.1.6.1)
+      activesupport (= 5.1.6.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.1.5)
-      activesupport (= 5.1.5)
+    activejob (5.1.6.1)
+      activesupport (= 5.1.6.1)
       globalid (>= 0.3.6)
-    activemodel (5.1.5)
-      activesupport (= 5.1.5)
-    activerecord (5.1.5)
-      activemodel (= 5.1.5)
-      activesupport (= 5.1.5)
+    activemodel (5.1.6.1)
+      activesupport (= 5.1.6.1)
+    activerecord (5.1.6.1)
+      activemodel (= 5.1.6.1)
+      activesupport (= 5.1.6.1)
       arel (~> 8.0)
     activerecord-import (0.25.0)
       activerecord (>= 3.2)
@@ -65,9 +65,9 @@ GEM
       multi_json (~> 1.11, >= 1.11.2)
       rack (>= 1.5.2, < 3)
       railties (>= 4.0, < 5.2)
-    activesupport (5.1.5)
+    activesupport (5.1.6.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     addressable (2.5.2)
@@ -105,9 +105,9 @@ GEM
       coffee-script
       execjs
       json
-    composite_primary_keys (10.0.3)
-      activerecord (~> 5.1.0, >= 5.1.5)
-    concurrent-ruby (1.0.5)
+    composite_primary_keys (10.0.5)
+      activerecord (~> 5.1.0, >= 5.1.6)
+    concurrent-ruby (1.1.3)
     coveralls (0.7.1)
       multi_json (~> 1.3)
       rest-client
@@ -199,7 +199,7 @@ GEM
     http-form_data (2.0.0)
     http_parser.rb (0.6.0)
     httpclient (2.8.3)
-    i18n (0.9.5)
+    i18n (1.1.1)
       concurrent-ruby (~> 1.0)
     icalendar (2.4.1)
     icalendar-recurrence (1.1.2)
@@ -225,7 +225,7 @@ GEM
     logging (2.2.2)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
-    loofah (2.2.2)
+    loofah (2.2.3)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     lumberjack (1.0.12)
@@ -233,10 +233,10 @@ GEM
       mime-types (>= 1.16, < 4)
     memoizable (0.4.2)
       thread_safe (~> 0.3, >= 0.3.1)
-    method_source (0.9.0)
-    mime-types (3.1)
+    method_source (0.9.2)
+    mime-types (3.2.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2016.0521)
+    mime-types-data (3.2018.0812)
     mini_portile2 (2.3.0)
     minitest (5.11.3)
     multi_json (1.12.2)
@@ -248,7 +248,7 @@ GEM
     nestful (1.1.3)
     net-ldap (0.16.1)
     netrc (0.11.0)
-    nio4r (2.3.0)
+    nio4r (2.3.1)
     nokogiri (1.8.5)
       mini_portile2 (~> 2.3.0)
     nori (2.6.0)
@@ -323,22 +323,22 @@ GEM
       pry (>= 0.9.11)
     public_suffix (3.0.1)
     puma (3.11.0)
-    rack (2.0.5)
+    rack (2.0.6)
     rack-livereload (0.3.16)
       rack
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (5.1.5)
-      actioncable (= 5.1.5)
-      actionmailer (= 5.1.5)
-      actionpack (= 5.1.5)
-      actionview (= 5.1.5)
-      activejob (= 5.1.5)
-      activemodel (= 5.1.5)
-      activerecord (= 5.1.5)
-      activesupport (= 5.1.5)
+    rails (5.1.6.1)
+      actioncable (= 5.1.6.1)
+      actionmailer (= 5.1.6.1)
+      actionpack (= 5.1.6.1)
+      actionview (= 5.1.6.1)
+      activejob (= 5.1.6.1)
+      activemodel (= 5.1.6.1)
+      activerecord (= 5.1.6.1)
+      activesupport (= 5.1.6.1)
       bundler (>= 1.3.0)
-      railties (= 5.1.5)
+      railties (= 5.1.6.1)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
@@ -347,9 +347,9 @@ GEM
       loofah (~> 2.2, >= 2.2.2)
     rails-observers (0.1.5)
       activemodel (>= 4.0)
-    railties (5.1.5)
-      actionpack (= 5.1.5)
-      activesupport (= 5.1.5)
+    railties (5.1.6.1)
+      actionpack (= 5.1.6.1)
+      activesupport (= 5.1.6.1)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
@@ -447,7 +447,7 @@ GEM
     therubyracer (0.12.3)
       libv8 (~> 3.16.14.15)
       ref
-    thor (0.20.0)
+    thor (0.20.3)
     thread_safe (0.3.6)
     tilt (2.0.8)
     tins (1.15.1)
@@ -563,7 +563,7 @@ DEPENDENCIES
   pry-stack_explorer
   puma
   rack-livereload
-  rails (= 5.1.5)
+  rails (= 5.1.6.1)
   rails-observers
   rb-fsevent
   rchardet (>= 1.8.0)

app/assets/javascripts/app/controllers/_application_controller_generic.coffee

@@ -1320,6 +1320,7 @@ class App.Import extends App.ControllerModal
     params.set('try', true)
     if _.isEmpty(params.get('data'))
       params.delete('data')
+    @formDisable(e)
     @ajax(
       id:          'csv_import'
       type:        'POST'
@@ -1341,6 +1342,14 @@ class App.Import extends App.ControllerModal
           return
         @data = data
         @update()
+        @formEnable(e)
+      error: (data) =>
+        details = data.responseJSON || {}
+        @notify
+          type:    'error'
+          msg:     App.i18n.translateContent(details.error_human || details.error || 'Unable to import!')
+          timeout: 6000
+        @formEnable(e)
     )
 
 class App.ImportTryResult extends App.ControllerModal
@@ -1361,10 +1370,23 @@ class App.ImportTryResult extends App.ControllerModal
       import_example_url: "#{@baseUrl}/import"
       result: @result
     ))
+
+    # check if data is processing...
+    if @data
+      result = App.view("#{@templateDirectory}/result")(
+        @data
+      )
+      content.find('.js-error').html(result)
+      if result
+        content.find('.js-error').removeClass('hide')
+      else
+        content.find('.js-error').addClass('hide')
+
     content
 
   onSubmit: (e) =>
     @params.set('try', false)
+    @formDisable(e)
     @ajax(
       id:          'csv_import'
       type:        'POST'
@@ -1386,6 +1408,14 @@ class App.ImportTryResult extends App.ControllerModal
           return
         @data = data
         @update()
+        @formEnable(e)
+      error: (data) =>
+        details = data.responseJSON || {}
+        @notify
+          type:    'error'
+          msg:     App.i18n.translateContent(details.error_human || details.error || 'Unable to import!')
+          timeout: 6000
+        @formEnable(e)
     )
 
 class App.ImportResult extends App.ControllerModal

app/assets/javascripts/app/controllers/_ui_element/ticket_perform_action.coffee

@@ -364,6 +364,11 @@ class App.UiElement.ticket_perform_action
           object: 'Ticket'
           display: 'Ticket'
         },
+        {
+          prefix: 'article'
+          object: 'TicketArticle'
+          display: 'Article'
+        },
         {
           prefix: 'user'
           object: 'User'

app/assets/javascripts/app/controllers/users.coffee

@@ -18,7 +18,7 @@ class Index extends App.ControllerSubContent
         { name: 'Import', 'data-type': 'import', class: 'btn' }
         { name: 'New User', 'data-type': 'new', class: 'btn--success' }
       ]
-      roles: App.Role.all()
+      roles: App.Role.findAllByAttribute('active', true)
     )
 
     @$('.tab').on(

app/assets/javascripts/app/lib/app_post/utils.coffee

@@ -709,16 +709,29 @@ class App.Utils
       key = key.replace(/<.+?>/g, '')
       levels  = key.split(/\./)
       dataRef = objects
+      dataRefLast = undefined
       for level in levels
-        if level of dataRef
+        if typeof dataRef is 'object' && level of dataRef
+          dataRefLast = dataRef
           dataRef = dataRef[level]
         else
           dataRef = ''
           break
+      value = undefined
       if typeof dataRef is 'function'
         value = dataRef()
       else if dataRef isnt undefined && dataRef isnt null && dataRef.toString
-        value = dataRef.toString()
+        if dataRefLast && dataRefLast.constructor && dataRefLast.constructor.className
+          localClassRef = App[dataRefLast.constructor.className]
+          if localClassRef && localClassRef.attributesGet
+            attributes = localClassRef.attributesGet()
+            if attributes && attributes[level]
+              if attributes[level]['tag'] is 'datetime'
+                value = App.i18n.translateTimestamp(dataRef)
+              else if attributes[level]['tag'] is 'date'
+                value = App.i18n.translateDate(dataRef)
+        if !value
+          value = dataRef.toString()
       else
         value = ''
       #console.log( "tag replacement #{key}, #{value} env: ", objects)
@@ -1146,7 +1159,7 @@ class App.Utils
 
     html.find('img').each( (index) ->
       src = $(@).attr('src')
-      if !src.match(/^data:/i)
+      if !src.match(/^(data|cid):/i) # <img src="cid: ..."> may mean broken emails (see issue #2305)
         base64 = App.Utils._htmlImage2DataUrl(@)
         $(@).attr('src', base64)
     )

app/assets/javascripts/app/lib/app_post/websocket.coffee

@@ -35,6 +35,11 @@ class App.WebSocket
       _instance ?= new _webSocketSingleton
     _instance.support()
 
+  @queue: ->
+    if _instance == undefined
+      _instance ?= new _webSocketSingleton
+    _instance.queue
+
 # The actual Singleton class
 class _webSocketSingleton extends App.Controller
   @include App.LogInclude
@@ -355,10 +360,10 @@ class _webSocketSingleton extends App.Controller
         success: (data) =>
           if data && data.error
             @client_id = undefined
-            @_ajaxInit( force: true )
+            @_ajaxInit(force: true)
         error: =>
           @client_id = undefined
-          @_ajaxInit( force: true )
+          @_ajaxInit(force: true)
       )
 
   _ajaxReceive: =>

app/assets/javascripts/app/views/ticket_overview/batch_dragger.jst.eco

@@ -1,3 +1,3 @@
-<div class="batch-dragger zIndex-10">
+<div class="batch-dragger">
 	<div class="batch-dragger-counter js-batch-dragger-count"></div>
 </div>

app/assets/stylesheets/zammad.scss

@@ -213,19 +213,6 @@ ol, ul {
   cursor: not-allowed;
 }
 
-.zIndex-1,
-.zIndex-2,
-.zIndex-3,
-.zIndex-4,
-.zIndex-5,
-.zIndex-6,
-.zIndex-7,
-.zIndex-8,
-.zIndex-9,
-.zIndex-10 {
-  position: relative;
-}
-
 .zIndex-1 {
   z-index: 100;
 }
@@ -2017,7 +2004,8 @@ input.has-error {
 }
 
 .selected-clue {
-  @extend .zIndex-6;
+  position: relative;
+  @extend .zIndex-7;
   pointer-events: none;
 }
 
@@ -3535,7 +3523,7 @@ footer {
     }
 
   .global-search-menu {
-    @extend .zIndex-6;
+    @extend .zIndex-7;
     background: #26272e;
     position: absolute;
     left: 0;
@@ -4031,7 +4019,7 @@ footer {
   padding: 10px 20px;
   overflow: auto;
   position: relative;
-  
+
   &--large-padding {
     padding: 30px 40px;
   }
@@ -4051,7 +4039,7 @@ footer {
 }
 
 .popover {
-  @extend .zIndex-6;
+  @extend .zIndex-7;
   position: absolute;
   font-family: inherit;
   width: 372px;
@@ -4173,6 +4161,7 @@ footer {
     max-width: 400px;
     min-width: 350px;
     flex-direction: column;
+    @extend .zIndex-10;
 
     &.is-visible {
       display: flex;
@@ -4682,18 +4671,18 @@ footer {
       left: 0;
       padding: 0 5px 0 20px;
       width: 100%;
-      
+
       .controls-label {
         display: none;
       }
-      
+
       .form-control {
         width: 70px;
         line-height: inherit;
         position: static;
         padding: 0;
         height: auto;
-        
+
         &.time {
           margin-left: 5px;
           width: 38px;
@@ -5410,7 +5399,7 @@ footer {
       color: inherit;
       align-items: center;
       border-bottom: 1px solid hsl(0,0%,96%);
-      
+
       &:last-child {
         border-bottom: none;
       }
@@ -5422,11 +5411,11 @@ footer {
         top: 9px;
         width: 38px;
         text-align: center;
-        
+
         svg {
           vertical-align: bottom;
         }
-        
+
         img {
           width: 30px;
           height: 30px;
@@ -6225,7 +6214,7 @@ footer {
 
 /* allow/show autocomplete in modal dialog */
 .ui-autocomplete.ui-widget-content {
-  @extend .zIndex-9;
+  @extend .zIndex-8;
   position: absolute;
 }
 
@@ -6235,11 +6224,11 @@ footer {
   color: white;
   font-family: inherit;
   font-size: inherit;
-  
+
   .ui-menu-item {
     padding: 9px 15px;
     list-style-image: none;
-    
+
     &:not(:first-child) {
       box-shadow: 0 1px rgba(255,255,255,.13) inset;
     }
@@ -6388,8 +6377,7 @@ footer {
   }
 
 .modal {
-  position: fixed;
-  @extend .zIndex-9;
+  @extend .zIndex-10;
 }
 
 .modal-dialog {
@@ -6397,7 +6385,7 @@ footer {
   max-width: 660px;
   margin-top: 35px;
   margin-bottom: 35px;
-  
+
   &.wizard {
     margin: 35px auto;
   }
@@ -6484,6 +6472,7 @@ footer {
 .modal.modal--local {
   display: block;
   padding: 0 40px;
+  @extend .zIndex-9;
 
   .modal-backdrop {
     background: hsla(210,17%,93%,.55);
@@ -6503,13 +6492,11 @@ footer {
 }
 
 .dropdown-backdrop {
-  position: fixed;
-  @extend .zIndex-5; // has to be behind dropdown-menu (dropdown-menu is also zIndex-5 but behind dropdown-backdrop in dom)
+  @extend .zIndex-5; // has to be behind dropdown-menu
 }
 
 .dropdown-menu {
-  @extend .zIndex-5; // has to be behind modal windows and beneath notifications (popover)
-  position: absolute;
+  @extend .zIndex-6; // has to be behind modal windows and beneath notifications (popover)
   margin: 0;
   padding: 0;
   min-width: 100%;
@@ -7383,7 +7370,7 @@ label + .wizard-buttonList {
   padding: 0;
   @extend %clickable;
   opacity: 0;
-  
+
   &:focus + label {
     text-decoration: underline;
   }
@@ -7683,7 +7670,7 @@ label + .wizard-buttonList {
 
 .horizontal-filter-value {
   flex-shrink: 0;
-  
+
   // lower the min-width of url input fields (normally 400px) so that it fits in
   input[type=url] {
     min-width: 200px;
@@ -8465,7 +8452,7 @@ output {
   top: 10px;
   left: 0;
   right: 0;
-  @extend .zIndex-10;
+  @extend .zIndex-9;
   pointer-events: none;
 
   .noty_bar {
@@ -9366,6 +9353,7 @@ output {
   width: 250px;
   height: 40px;
   will-change: transform;
+  @extend .zIndex-10;
 
   &-item {
     position: absolute;

app/controllers/concerns/ticket_stats.rb

@@ -8,6 +8,8 @@ module TicketStats
       limit: limit,
       condition: condition,
       current_user: current_user,
+      sort_by: 'created_at',
+      order_by: 'desc',
     )
     assets_of_tickets(tickets, assets)
   end

app/controllers/long_polling_controller.rb

@@ -2,6 +2,7 @@
 
 class LongPollingController < ApplicationController
   skip_before_action :session_update # prevent race conditions
+  prepend_before_action :authentication_check_only
 
   # GET /api/v1/message_send
   def message_send
@@ -14,24 +15,22 @@ class LongPollingController < ApplicationController
       client_id = client_id_gen
       log 'new client connection', client_id
     end
-    if !params['data']
-      params['data'] = {}
-    end
+    data = params['data'].permit!.to_h
     session_data = {}
     if current_user&.id
       session_data = { 'id' => current_user.id }
     end
 
     # spool messages for new connects
-    if params['data']['spool']
-      Sessions.spool_create(params['data'])
+    if data['spool']
+      Sessions.spool_create(data)
     end
-    if params['data']['event'] == 'login'
+    if data['event'] == 'login'
       Sessions.create(client_id, session_data, { type: 'ajax' })
-    elsif params['data']['event']
+    elsif data['event']
       message = Sessions::Event.run(
-        event: params['data']['event'],
-        payload: params['data'],
+        event: data['event'],
+        payload: data,
         session: session_data,
         client_id: client_id,
         clients: {},
@@ -41,15 +40,15 @@ class LongPollingController < ApplicationController
         Sessions.send(client_id, message)
       end
     else
-      log "unknown message '#{params['data'].inspect}'", client_id
+      log "unknown message '#{data.inspect}'", client_id
     end
 
     if new_connection
       result = { client_id: client_id }
       render json: result
-    else
-      render json: {}
+      return
     end
+    render json: {}
   end
 
   # GET /api/v1/message_receive
@@ -110,7 +109,7 @@ class LongPollingController < ApplicationController
     params[:client_id].to_s
   end
 
-  def log( data, client_id = '-' )
+  def log(data, client_id = '-')
     logger.info "client(#{client_id}) #{data}"
   end
 end

app/controllers/organizations_controller.rb

@@ -349,7 +349,12 @@ curl http://localhost/api/v1/organization/{id} -v -u #{login}:#{password} -H "Co
   # @response_message 401 Invalid session.
   def import_start
     permission_check('admin.user')
-    string = params[:data] || params[:file].read.force_encoding('utf-8')
+    string = params[:data]
+    if string.blank? && params[:file].present?
+      string = params[:file].read.force_encoding('utf-8')
+    end
+    raise Exceptions::UnprocessableEntity, 'No source data submitted!' if string.blank?
+
     result = Organization.csv_import(
       string: string,
       parse_params: {

app/controllers/sessions_controller.rb

@@ -2,7 +2,7 @@
 
 class SessionsController < ApplicationController
   prepend_before_action :authentication_check, only: %i[switch_to_user list delete]
-  skip_before_action :verify_csrf_token, only: %i[create show destroy create_omniauth failure_omniauth create_sso]
+  skip_before_action :verify_csrf_token, only: %i[show destroy create_omniauth failure_omniauth create_sso]
 
   # "Create" a login, aka "log the user in"
   def create

app/controllers/text_modules_controller.rb

@@ -186,7 +186,12 @@ curl http://localhost/api/v1/text_modules.json -v -u #{login}:#{password} -H "Co
   # @response_message 401 Invalid session.
   def import_start
     permission_check('admin.text_module')
-    string = params[:data] || params[:file].read.force_encoding('utf-8')
+    string = params[:data]
+    if string.blank? && params[:file].present?
+      string = params[:file].read.force_encoding('utf-8')
+    end
+    raise Exceptions::UnprocessableEntity, 'No source data submitted!' if string.blank?
+
     result = TextModule.csv_import(
       string: string,
       parse_params: {

app/controllers/ticket_articles_controller.rb

@@ -318,8 +318,14 @@ class TicketArticlesController < ApplicationController
       raise 'Only can import tickets if system is in import mode.'
     end
 
+    string = params[:data]
+    if string.blank? && params[:file].present?
+      string = params[:file].read.force_encoding('utf-8')
+    end
+    raise Exceptions::UnprocessableEntity, 'No source data submitted!' if string.blank?
+
     result = Ticket::Article.csv_import(
-      string: params[:file].read.force_encoding('utf-8'),
+      string: string,
       parse_params: {
         col_sep: ';',
       },

app/controllers/tickets_controller.rb

@@ -641,7 +641,12 @@ class TicketsController < ApplicationController
       raise 'Only can import tickets if system is in import mode.'
     end
 
-    string = params[:data] || params[:file].read.force_encoding('utf-8')
+    string = params[:data]
+    if string.blank? && params[:file].present?
+      string = params[:file].read.force_encoding('utf-8')
+    end
+    raise Exceptions::UnprocessableEntity, 'No source data submitted!' if string.blank?
+
     result = Ticket.csv_import(
       string: string,
       parse_params: {

app/controllers/users_controller.rb

@@ -1043,7 +1043,12 @@ curl http://localhost/api/v1/users/avatar -v -u #{login}:#{password} -H "Content
   # @response_message 401 Invalid session.
   def import_start
     permission_check('admin.user')
-    string = params[:data] || params[:file].read.force_encoding('utf-8')
+    string = params[:data]
+    if string.blank? && params[:file].present?
+      string = params[:file].read.force_encoding('utf-8')
+    end
+    raise Exceptions::UnprocessableEntity, 'No source data submitted!' if string.blank?
+
     result = User.csv_import(
       string: string,
       parse_params: {

app/models/application_model/can_lookup.rb

@@ -20,20 +20,33 @@ returns
 =end
 
     def lookup(**attr)
+      raise ArgumentError, "Multiple lookup attributes given (#{attr.keys.join(', ')}), only support (#{lookup_keys.join(', ')})" if attr.many?
+
       attr.transform_keys!(&:to_sym).slice!(*lookup_keys)
       raise ArgumentError, "Valid lookup attribute required (#{lookup_keys.join(', ')})" if attr.empty?
-      raise ArgumentError, "Multiple lookup attributes given (#{attr.keys.join(', ')})" if attr.many?
 
       record   = cache_get(attr.values.first)
       record ||= find_and_save_to_cache_by(attr)
     end
 
-    private
+=begin
+
+return possible lookup keys for model
+
+  result = Model.lookup_keys
+
+returns
+
+  [:id, :name] # or fror users [:id, :login, :email]
+
+=end
 
     def lookup_keys
       @lookup_keys ||= %i[id name login email number] & attribute_names.map(&:to_sym)
     end
 
+    private
+
     def find_and_save_to_cache_by(attr)
       if !Rails.application.config.db_case_sensitive && string_key?(attr.keys.first)
         where(attr).find { |r| r[attr.keys.first] == attr.values.first }

app/models/channel/driver/imap.rb

@@ -94,32 +94,40 @@ example
     Rails.logger.info "fetching imap (#{options[:host]}/#{options[:user]} port=#{port},ssl=#{ssl},starttls=#{starttls},folder=#{folder},keep_on_server=#{keep_on_server})"
 
     # on check, reduce open_timeout to have faster probing
-    timeout = 45
+    check_type_timeout = 45
     if check_type == 'check'
-      timeout = 6
+      check_type_timeout = 6
     end
 
-    Timeout.timeout(timeout) do
+    timeout(check_type_timeout) do
       @imap = ::Net::IMAP.new(options[:host], port, ssl, nil, false)
       if starttls
         @imap.starttls()
       end
     end
 
-    @imap.login(options[:user], options[:password])
+    timeout(check_type_timeout) do
+      @imap.login(options[:user], options[:password])
+    end
 
-    # select folder
-    @imap.select(folder)
+    timeout(check_type_timeout) do
+      # select folder
+      @imap.select(folder)
+    end
 
     # sort messages by date on server (if not supported), if not fetch messages via search (first in, first out)
     filter = ['ALL']
     if keep_on_server && check_type != 'check' && check_type != 'verify'
       filter = %w[NOT SEEN]
     end
-    begin
-      message_ids = @imap.sort(['DATE'], filter, 'US-ASCII')
-    rescue
-      message_ids = @imap.search(filter)
+
+    message_ids = nil
+    timeout(6.minutes) do
+      begin
+        message_ids = @imap.sort(['DATE'], filter, 'US-ASCII')
+      rescue
+        message_ids = @imap.search(filter)
+      end
     end
 
     # check mode only
@@ -131,7 +139,10 @@ example
       # check messages
       message_ids.each do |message_id|
 
-        message_meta = @imap.fetch(message_id, ['RFC822.HEADER'])[0].attr
+        message_meta = nil
+        timeout(1.minute) do
+          message_meta = @imap.fetch(message_id, ['RFC822.HEADER'])[0].attr
+        end
 
         # check how many content messages we have, for notice used
         header = message_meta['RFC822.HEADER']
@@ -158,7 +169,10 @@ example
       # check for verify message
       message_ids.each do |message_id|
 
-        message_meta = @imap.fetch(message_id, ['ENVELOPE'])[0].attr
+        message_meta = nil
+        timeout(1.minute) do
+          message_meta = @imap.fetch(message_id, ['ENVELOPE'])[0].attr
+        end
 
         # check if verify message exists
         subject = message_meta['ENVELOPE'].subject
@@ -166,8 +180,10 @@ example
         next if subject !~ /#{verify_string}/
 
         Rails.logger.info " - verify email #{verify_string} found"
-        @imap.store(message_id, '+FLAGS', [:Deleted])
-        @imap.expunge()
+        timeout(600) do
+          @imap.store(message_id, '+FLAGS', [:Deleted])
+          @imap.expunge()
+        end
         disconnect
         return {
           result: 'ok',
@@ -189,7 +205,10 @@ example
       count += 1
       Rails.logger.info " - message #{count}/#{count_all}"
 
-      message_meta = @imap.fetch(message_id, ['RFC822.SIZE', 'ENVELOPE', 'FLAGS', 'INTERNALDATE'])[0]
+      message_meta = nil
+      timeout(1.minute) do
+        message_meta = @imap.fetch(message_id, ['RFC822.SIZE', 'ENVELOPE', 'FLAGS', 'INTERNALDATE'])[0]
+      end
 
       # ignore to big messages
       info = too_big?(message_meta, count, count_all)
@@ -205,19 +224,28 @@ example
       next if already_imported?(message_id, message_meta, count, count_all, keep_on_server, channel)
 
       # delete email from server after article was created
-      msg = @imap.fetch(message_id, 'RFC822')[0].attr['RFC822']
+      msg = nil
+      timeout(1.minute) do
+        msg = @imap.fetch(message_id, 'RFC822')[0].attr['RFC822']
+      end
       next if !msg
 
       process(channel, msg, false)
-      if !keep_on_server
-        @imap.store(message_id, '+FLAGS', [:Deleted])
-      else
-        @imap.store(message_id, '+FLAGS', [:Seen])
+
+      timeout(1.minute) do
+        if !keep_on_server
+          @imap.store(message_id, '+FLAGS', [:Deleted])
+        else
+          @imap.store(message_id, '+FLAGS', [:Seen])
+        end
       end
       count_fetched += 1
     end
+
     if !keep_on_server
-      @imap.expunge()
+      timeout(10.minutes) do
+        @imap.expunge()
+      end
     end
     disconnect
     if count.zero?
@@ -234,7 +262,9 @@ example
   def disconnect
     return if !@imap
 
-    @imap.disconnect()
+    timeout(1.minute) do
+      @imap.disconnect()
+    end
   end
 
 =begin
@@ -273,7 +303,9 @@ returns
       return false if ticket.preferences[:channel_id] != channel[:id]
     end
 
-    @imap.store(message_id, '+FLAGS', [:Seen])
+    timeout(1.minute) do
+      @imap.store(message_id, '+FLAGS', [:Seen])
+    end
     Rails.logger.info "  - ignore message #{count}/#{count_all} - because message message id already imported"
     true
   end
@@ -296,4 +328,10 @@ returns
     false
   end
 
+  def timeout(seconds)
+    Timeout.timeout(seconds) do
+      yield
+    end
+  end
+
 end

app/models/concerns/can_csv_import.rb

@@ -118,6 +118,17 @@ returns
         return result
       end
 
+      # check if min one lookup key exists
+      if header.count == (header - lookup_keys.map(&:to_s)).count
+        errors.push "No lookup column like #{lookup_keys.map(&:to_s).join(',')} for #{new.class} found."
+        result = {
+          errors: errors,
+          try: try,
+          result: 'failed',
+        }
+        return result
+      end
+
       # get payload based on csv
       payload = []
       rows.each do |row|
@@ -172,11 +183,15 @@ returns
       payload.each do |attributes|
         line_count += 1
         record = nil
-        %i[id number name login email].each do |lookup_by|
-          next if !attributes[lookup_by]
+        lookup_keys.each do |lookup_by|
+          next if attributes[lookup_by].blank?
 
           params = {}
-          params[lookup_by] = attributes[lookup_by]
+          params[lookup_by] = if %i[email login].include?(lookup_by)
+                                attributes[lookup_by].downcase
+                              else
+                                attributes[lookup_by]
+                              end
           record = lookup(params)
           break if record
         end
@@ -199,6 +214,7 @@ returns
         end
 
         # create object
+        BulkImportInfo.enable
         Transaction.execute(disable_notification: true, reset_user_id: true) do
           UserInfo.current_user_id = clean_params[:updated_by_id] || clean_params[:created_by_id]
           if !record || delete == true
@@ -217,7 +233,7 @@ returns
               record.associations_from_param(attributes)
               record.save!
             rescue => e
-              errors.push "Line #{line_count}: #{e.message}"
+              errors.push "Line #{line_count}: Unable to create record - #{e.message}"
               next
             end
           else
@@ -234,14 +250,20 @@ returns
 
               record.with_lock do
                 record.associations_from_param(attributes)
-                record.update!(clean_params)
+                clean_params.each do |key, value|
+                  record[key] = value
+                end
+                next if !record.changed?
+
+                record.save!
               end
             rescue => e
-              errors.push "Line #{line_count}: #{e.message}"
+              errors.push "Line #{line_count}: Unable to update record - #{e.message}"
               next
             end
           end
         end
+        BulkImportInfo.disable
 
         records.push record
       end
@@ -258,7 +280,6 @@ returns
         try: try,
         result: result,
       }
-
     end
 
 =begin

app/models/cti/caller_id.rb

@@ -9,6 +9,9 @@ module Cti
     after_commit :update_cti_logs, on: :destroy
     after_commit :update_cti_logs_with_fg_optimization, on: :create
 
+    skip_callback :commit, :after, :update_cti_logs, if: -> { BulkImportInfo.enabled? }
+    skip_callback :commit, :after, :update_cti_logs_with_fg_optimization, if: -> { BulkImportInfo.enabled? }
+
 =begin
 
   Cti::CallerId.maybe_add(

app/models/setting.rb

@@ -125,7 +125,7 @@ reload config settings
     end
 
     @@change_id = Cache.get('Setting::ChangeId') # rubocop:disable Style/ClassVars
-    @@lookup_at = Time.zone.now # rubocop:disable Style/ClassVars
+    @@lookup_at = Time.now.to_i # rubocop:disable Style/ClassVars
     true
   end
   private_class_method :load
@@ -147,14 +147,14 @@ reload config settings
 
   # check if cache is still valid
   def self.cache_valid?
-    if @@lookup_at && @@lookup_at > Time.zone.now - @@lookup_timeout
+    if @@lookup_at && @@lookup_at > Time.now.to_i - @@lookup_timeout
       #logger.debug "Setting.cache_valid?: cache_id has been set within last #{@@lookup_timeout} seconds"
       return true
     end
 
     change_id = Cache.get('Setting::ChangeId')
     if @@change_id && change_id == @@change_id
-      @@lookup_at = Time.zone.now # rubocop:disable Style/ClassVars
+      @@lookup_at = Time.now.to_i # rubocop:disable Style/ClassVars
       #logger.debug "Setting.cache_valid?: cache still valid, #{@@change_id}/#{change_id}"
       return true
     end

app/models/user.rb

@@ -21,12 +21,15 @@ class User < ApplicationModel
 
   before_validation :check_name, :check_email, :check_login, :ensure_uniq_email, :ensure_password, :ensure_roles, :ensure_identifier
   before_validation :check_mail_delivery_failed, on: :update
-  before_create   :check_preferences_default, :validate_preferences, :validate_ooo, :domain_based_assignment, :set_locale
-  before_update   :check_preferences_default, :validate_preferences, :validate_ooo, :reset_login_failed, :validate_agent_limit_by_attributes, :last_admin_check_by_attribute
-  after_create    :avatar_for_email_check
-  after_update    :avatar_for_email_check
-  after_commit    :update_caller_id
-  before_destroy  :destroy_longer_required_objects
+  before_create     :check_preferences_default, :validate_preferences, :validate_ooo, :domain_based_assignment, :set_locale
+  before_update     :check_preferences_default, :validate_preferences, :validate_ooo, :reset_login_failed, :validate_agent_limit_by_attributes, :last_admin_check_by_attribute
+  after_create      :avatar_for_email_check
+  after_update      :avatar_for_email_check
+  after_commit      :update_caller_id
+  before_destroy    :destroy_longer_required_objects
+
+  skip_callback :create, :after, :avatar_for_email_check, if: -> { BulkImportInfo.enabled? }
+  skip_callback :update, :after, :avatar_for_email_check, if: -> { BulkImportInfo.enabled? }
 
   store :preferences
 

config/initializers/delayed_jobs_verify_db_connection.rb

@@ -0,0 +1,10 @@
+require 'delayed_job'
+
+module Delayed
+  class Job < ::ActiveRecord::Base
+
+    def self.recover_from(_error)
+      ::ActiveRecord::Base.connection.verify!
+    end
+  end
+end

config/initializers/filter_parameter_logging.rb

@@ -1,4 +1,4 @@
 # Be sure to restart your server when you modify this file.
 
 # Configure sensitive parameters which will be filtered from the log file.
-Rails.application.config.filter_parameters += %i[password bind_pw]
+Rails.application.config.filter_parameters += %i[password bind_pw state.body state.article.body]

lib/bulk_import_info.rb

@@ -0,0 +1,13 @@
+module BulkImportInfo
+  def self.enabled?
+    Thread.current[:bulk_import]
+  end
+
+  def self.enable
+    Thread.current[:bulk_import] = true
+  end
+
+  def self.disable
+    Thread.current[:bulk_import] = false
+  end
+end

lib/core_ext/string.rb

@@ -125,11 +125,13 @@ class String
     link_list = ''
     counter   = 0
     if !string_only
-      string.gsub!(/<a[[:space:]].*?href=("|')(.+?)("|').*?>/ix) do
-        link = $2
-        counter = counter + 1
-        link_list += "[#{counter}] #{link}\n"
-        "[#{counter}] "
+      if string.scan(/<a[[:space:]]/i).count < 5_000
+        string.gsub!(/<a[[:space:]].*?href=("|')(.+?)("|').*?>/ix) do
+          link = $2
+          counter = counter + 1
+          link_list += "[#{counter}] #{link}\n"
+          "[#{counter}] "
+        end
       end
     else
       string.gsub!(%r{<a[[:space:]]+(|\S+[[:space:]]+)href=("|')(.+?)("|')([[:space:]]*|[[:space:]]+[^>]*)>(.+?)<[[:space:]]*/a[[:space:]]*>}mxi) do |_placeholder|
@@ -477,7 +479,21 @@ class String
   def utf8_encode!(**options)
     return force_encoding('utf-8') if dup.force_encoding('utf-8').valid_encoding?
 
-    viable_encodings(try_first: options[:from]).each do |enc|
+    # convert string to given charset, if valid_encoding? is true
+    if options[:from].present?
+      begin
+        encoding = Encoding.find(options[:from])
+        if encoding.present? && dup.force_encoding(encoding).valid_encoding?
+          force_encoding(encoding)
+          return encode!('utf-8', encoding)
+        end
+      rescue ArgumentError, EncodingError => e
+        Rails.logger.error { e.inspect }
+      end
+    end
+
+    # try to find valid encodings of string
+    viable_encodings.each do |enc|
       begin
         return encode!('utf-8', enc)
       rescue EncodingError => e

lib/html_sanitizer.rb

@@ -10,6 +10,7 @@ satinize html string based on whiltelist
 =end
 
   def self.strict(string, external = false)
+    @fqdn = Setting.get('fqdn')
 
     # config
     tags_remove_content = Rails.configuration.html_sanitizer_tags_remove_content
@@ -30,7 +31,7 @@ satinize html string based on whiltelist
     scrubber_link = Loofah::Scrubber.new do |node|
 
       # wrap plain-text URLs in <a> tags
-      if node.is_a?(Nokogiri::XML::Text) && node.ancestors.map(&:name).exclude?('a')
+      if node.is_a?(Nokogiri::XML::Text) && node.content.present? && node.content.include?(':') && node.ancestors.map(&:name).exclude?('a')
         urls = URI.extract(node.content, LINKABLE_URL_SCHEMES)
                   .map { |u| u.sub(/[,.]$/, '') }      # URI::extract captures trailing dots/commas
                   .reject { |u| u.match?(/^[^:]+:$/) } # URI::extract will match, e.g., 'tel:'
@@ -396,9 +397,10 @@ cleanup html string:
   end
 
   def self.sanitize_attachment_disposition(url)
+    @fqdn ||= Setting.get('fqdn')
     uri = URI(url)
 
-    if uri.host == Setting.get('fqdn') && uri.query.present?
+    if uri.host == @fqdn && uri.query.present?
       params = CGI.parse(uri.query || '')
                   .tap { |p| p.merge!('disposition' => 'attachment') if p.include?('disposition') }
       uri.query = URI.encode_www_form(params)
@@ -432,6 +434,7 @@ reolace inline images with cid images
 =end
 
   def self.replace_inline_images(string, prefix = rand(999_999_999))
+    fqdn = Setting.get('fqdn')
     attachments_inline = []
     filename_counter = 0
     scrubber = Loofah::Scrubber.new do |node|
@@ -439,7 +442,7 @@ reolace inline images with cid images
         if node['src'] && node['src'] =~ %r{^(data:image/(jpeg|png);base64,.+?)$}i
           filename_counter += 1
           file_attributes = StaticAssets.data_url_attributes($1)
-          cid = "#{prefix}.#{rand(999_999_999)}@#{Setting.get('fqdn')}"
+          cid = "#{prefix}.#{rand(999_999_999)}@#{fqdn}"
           filename = cid
           if file_attributes[:file_extention].present?
             filename = "image#{filename_counter}.#{file_attributes[:file_extention]}"

lib/sequencer/unit/import/zendesk/sub_sequence/base.rb

@@ -39,6 +39,10 @@ class Sequencer
 
             def resource_iteration(&block)
               resource_collection.public_send(resource_iteration_method, &block)
+            rescue ZendeskAPI::Error::NetworkError => e
+              return if e.response.status.to_s == '403' && resource_klass.in?(%w[UserField OrganizationField])
+
+              raise
             end
 
             def resource_collection

lib/sessions.rb

@@ -334,6 +334,15 @@ send message to recipient client
 
   Sessions.send_to(user_id, data)
 
+e. g.
+
+  Sessions.send_to(user_id, {
+    event: 'session:takeover',
+    data: {
+      taskbar_id: 12312
+    },
+  })
+
 returns
 
   true|false
@@ -476,6 +485,14 @@ remove all session and spool messages
     true
   end
 
+=begin
+
+create spool messages
+
+  Sessions.spool_create(some: 'data')
+
+=end
+
   def self.spool_create(data)
     msg = JSON.generate(data)
     path = "#{@path}/spool/"
@@ -492,6 +509,14 @@ remove all session and spool messages
     end
   end
 
+=begin
+
+get spool messages
+
+  Sessions.spool_list(junger_then, for_user_id)
+
+=end
+
   def self.spool_list(timestamp, current_user_id)
     path = "#{@path}/spool/"
     FileUtils.mkpath path
@@ -512,6 +537,7 @@ remove all session and spool messages
         file.flock(File::LOCK_SH)
         message = file.read
         file.flock(File::LOCK_UN)
+        message_parsed = {}
         begin
           spool = JSON.parse(message)
           message_parsed = JSON.parse(spool['msg'])
@@ -530,7 +556,7 @@ remove all session and spool messages
         # add spool attribute to push spool info to clients
         message_parsed['spool'] = true
 
-        # only send not already now messages
+        # only send not already older messages
         if !timestamp || timestamp < spool['timestamp']
 
           # spool to recipient list
@@ -573,6 +599,19 @@ remove all session and spool messages
     data
   end
 
+=begin
+
+delete spool messages
+
+  Sessions.spool_delete
+
+=end
+
+  def self.spool_delete
+    path = "#{@path}/spool/"
+    FileUtils.rm_rf path
+  end
+
   def self.jobs(node_id = nil)
 
     # just make sure that spool path exists

lib/sessions/event.rb

@@ -7,7 +7,8 @@ class Sessions::Event
     begin
       backend = load_adapter(adapter)
     rescue => e
-      return { event: 'error', data: { error: "No such event #{params[:event]}", payload: params[:payload] } }
+      Rails.logger.error e
+      return { event: 'error', data: { error: "No such event #{params[:event]}: #{e.inspect}", payload: params[:payload] } }
     end
 
     begin
@@ -16,6 +17,7 @@ class Sessions::Event
       instance.destroy
       result
     rescue => e
+      Rails.logger.error e
       return { event: 'error', data: { error: e.message, payload: params[:payload] } }
     end
   end

lib/sessions/event/base.rb

@@ -9,6 +9,14 @@ class Sessions::Event::Base
     return if !@clients[@client_id]
 
     @is_web_socket = true
+
+    return if !self.class.instance_variable_get(:@database_connection)
+
+    ActiveRecord::Base.establish_connection
+  end
+
+  def self.inherited(subclass)
+    subclass.instance_variable_set(:@database_connection, @database_connection)
   end
 
   def websocket_send(recipient_client_id, data)
@@ -120,8 +128,18 @@ class Sessions::Event::Base
     puts "#{Time.now.utc.iso8601}:client(#{client_id}) #{data}"
     #puts "#{Time.now.utc.iso8601}:#{ level }:client(#{ client_id }) #{ data }"
     # rubocop:enable Rails/Output
+    #Rails.logger.info "#{Time.now.utc.iso8601}:client(#{client_id}) #{data}"
   end
 
-  def destroy; end
+  def self.database_connection_required
+    @database_connection = true
+  end
+
+  def destroy
+    return if !@is_web_socket
+    return if !self.class.instance_variable_get(:@database_connection)
+
+    ActiveRecord::Base.remove_connection
+  end
 
 end

lib/sessions/event/broadcast.rb

@@ -1,5 +1,15 @@
 class Sessions::Event::Broadcast < Sessions::Event::Base
 
+=begin
+
+Event module to broadcast messages to all client connections.
+
+To execute this manually, just paste the following into the browser console
+
+  App.WebSocket.send({event:'broadcast', recipient: { user_id: [1,2,3]}, data: {some: 'key'}})
+
+=end
+
   def run
 
     # list all current clients

lib/sessions/event/chat_base.rb

@@ -1,17 +1,5 @@
 class Sessions::Event::ChatBase < Sessions::Event::Base
-
-  def initialize(params)
-    super(params)
-    return if !@is_web_socket
-
-    ActiveRecord::Base.establish_connection
-  end
-
-  def destroy
-    return if !@is_web_socket
-
-    ActiveRecord::Base.remove_connection
-  end
+  database_connection_required
 
   def run
 

lib/sessions/event/login.rb

@@ -1,12 +1,20 @@
 class Sessions::Event::Login < Sessions::Event::Base
+  database_connection_required
+
+=begin
+
+Event module to start websocket session for new client connections.
+
+To execute this manually, just paste the following into the browser console
+
+  App.WebSocket.send({event:'login', session_id: '123'})
+
+=end
 
   def run
 
     # get user_id
     session = nil
-    if @is_web_socket
-      ActiveRecord::Base.establish_connection
-    end
 
     app_version = AppVersion.event_data
 
@@ -14,12 +22,7 @@ class Sessions::Event::Login < Sessions::Event::Base
       session = ActiveRecord::SessionStore::Session.find_by(session_id: @payload['session_id'])
     end
 
-    if @is_web_socket
-      ActiveRecord::Base.remove_connection
-    end
-
     new_session_data = {}
-
     if session&.data && session.data['user_id']
       new_session_data = {
         'id' => session.data['user_id'],

lib/sessions/event/maintenance.rb

@@ -1,17 +1,15 @@
 class Sessions::Event::Maintenance < Sessions::Event::Base
+  database_connection_required
 
-  def initialize(params)
-    super(params)
-    return if !@is_web_socket
+=begin
 
-    ActiveRecord::Base.establish_connection
-  end
+Event module to broadcast maintenance messages to all client connections.
 
-  def destroy
-    return if !@is_web_socket
+To execute this manually, just paste the following into the browser console
 
-    ActiveRecord::Base.remove_connection
-  end
+  App.WebSocket.send({event:'maintenance', data: {some: 'key'}})
+
+=end
 
   def run
 

lib/sessions/event/ping.rb

@@ -1,5 +1,15 @@
 class Sessions::Event::Ping < Sessions::Event::Base
 
+=begin
+
+Event module to send pong to client connection.
+
+To execute this manually, just paste the following into the browser console
+
+  App.WebSocket.send({event:'ping'})
+
+=end
+
   def run
     {
       event: 'pong',

lib/sessions/event/spool.rb

@@ -1,6 +1,15 @@
 class Sessions::Event::Spool < Sessions::Event::Base
 
-  # get spool messages and send them to new client connection
+=begin
+
+Event module to serve spool messages and send them to new client connection.
+
+To execute this manually, just paste the following into the browser console
+
+  App.WebSocket.send({event:'spool'})
+
+=end
+
   def run
 
     # error handling
@@ -11,8 +20,13 @@ class Sessions::Event::Spool < Sessions::Event::Base
     end
 
     if !@session || !@session['id']
-      log 'error', "can't send spool, session not authenticated"
-      return
+      log 'error', "Can't send spool, session not authenticated"
+      return {
+        event: 'error',
+        data: {
+          error: 'Can\'t send spool, session not authenticated',
+        },
+      }
     end
 
     spool = Sessions.spool_list(@payload['timestamp'], @session['id'])

lib/sessions/event/ticket_overview_index.rb

@@ -1,4 +1,5 @@
 class Sessions::Event::TicketOverviewIndex < Sessions::Event::Base
+  database_connection_required
 
   def run
     return if !valid_session?

lib/sessions/event/ticket_overview_list.rb

@@ -1,4 +1,5 @@
 class Sessions::Event::TicketOverviewList < Sessions::Event::Base
+  database_connection_required
 
   def run
     return if !valid_session?

lib/sessions/event/who_am_i.rb

@@ -0,0 +1,46 @@
+class Sessions::Event::WhoAmI < Sessions::Event::Base
+  database_connection_required
+
+=begin
+
+Event module to send `who am i` to client connection.
+
+To execute this manually, just paste the following into the browser console
+
+  App.WebSocket.send({event:'who_am_i'})
+
+=end
+
+  def run
+
+    if !@session || !@session['id']
+      return {
+        event: 'who_am_i',
+        data: {
+          message: 'session not authenticated',
+        },
+      }
+    end
+
+    user = User.find_by(id: @session['id'])
+
+    if !user
+      return {
+        event: 'who_am_i',
+        data: {
+          message: "No such user with id #{@session['id']}",
+        },
+      }
+    end
+    attributes = user.attributes
+    attributes.delete('password')
+    {
+      event: 'who_am_i',
+      data: {
+        message: 'session authenticated',
+        user: attributes,
+      },
+    }
+  end
+
+end

lib/tasks/zammad/ci/settings.rake

@@ -11,7 +11,7 @@ namespace :zammad do
 
       next if args[:elasticsearch] != 'with_elasticsearch'
 
-      Setting.set('es_url', 'http://localhost:9200')
+      Setting.set('es_url', 'http://elasticsearch:9200')
       Setting.set('es_index', "browser_test_#{ENV['CI_BUILD_ID']}")
 
       Rake::Task['searchindex:rebuild'].invoke

public/assets/tests/html_utils.js

@@ -1246,7 +1246,7 @@ test("identify signature by HTML", function() {
 });
 
 // check attachment references
-test("check replace tags", function() {
+test("check check attachment reference", function() {
   var message = 'some not existing'
   var result = false
   var verify = App.Utils.checkAttachmentReference(message)
@@ -1392,6 +1392,35 @@ test("check replace tags", function() {
   }
   verify = App.Utils.replaceTags(message, data)
   equal(verify, result)
+
+  user = new App.User({
+    firstname: 'Bob',
+    lastname: 'Smith',
+    created_at: '2018-10-31T10:00:00Z',
+  })
+  message = "<div>#{user.firstname} #{user.created_at}</div>"
+  result  = '<div>Bob 10/31/2018 10:00</div>'
+  data    = {
+    user: user
+  }
+  verify = App.Utils.replaceTags(message, data)
+  equal(verify, result)
+
+  message = "<div>#{user.firstname} #{user.created_at.date}</div>"
+  result  = '<div>Bob -</div>'
+  data    = {
+    user: user
+  }
+  verify = App.Utils.replaceTags(message, data)
+  equal(verify, result)
+
+  message = "<div>#{user.firstname} #{user.created.date}</div>"
+  result  = '<div>Bob -</div>'
+  data    = {
+    user: user
+  }
+  verify = App.Utils.replaceTags(message, data)
+  equal(verify, result)
 });
 
 // check attibute validation
@@ -3079,6 +3108,11 @@ test("htmlImage2DataUrl", function() {
   result = App.Utils.htmlImage2DataUrl(source)
   equal(result, should, source)
 
+  source = '<img src="cid:1234">some test'
+  should = '<img src="cid:1234">some test'
+  result = App.Utils.htmlImage2DataUrl(source)
+  equal(result, should, source)
+
 });
 
 source = '<img src="/assets/images/avatar-bg.png">some test'

public/assets/tests/ui.js

@@ -241,7 +241,7 @@ test("check pretty date", function() {
 
     result = App.PrettyDate.humanTime(current.getTime() + (60050 * 60 * 24 * 5.5), escalation, long, type);
     diff = 60 * 60 * 24 * 5.5
-    equal(result, getAbsolute(new Date(current.getTime() + (60050 * 60 * 24 * 5.5)), diff), 'in 30.5 days')
+    equal(result, getAbsolute(new Date(current.getTime() + (60050 * 60 * 24 * 5.5)), diff), 'in 5.5 days')
 
     result = App.PrettyDate.humanTime(current.getTime() + (60050 * 60 * 24 * 30.5), escalation, long, type);
     diff = (60 * 60 * 24 * 30.5);
@@ -323,7 +323,7 @@ test("check pretty date", function() {
     equal(result, getTimestamp(new Date(current.getTime() + (60050 * 60 * 24 * 2.5))), 'in 2.5 days')
 
     result = App.PrettyDate.humanTime(current.getTime() + (60050 * 60 * 24 * 5.5), escalation, long, type);
-    equal(result, getTimestamp(new Date(current.getTime() + (60050 * 60 * 24 * 5.5))), 'in 30.5 days')
+    equal(result, getTimestamp(new Date(current.getTime() + (60050 * 60 * 24 * 5.5))), 'in 5.5 days')
 
     result = App.PrettyDate.humanTime(current.getTime() + (60050 * 60 * 24 * 30.5), escalation, long, type);
     equal(result, getTimestamp(new Date(current.getTime() + 60050 * 60 * 24 * 30.5)), 'in 30.5 days')
@@ -337,14 +337,20 @@ test("check pretty date", function() {
     months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
     month = months[date.getMonth()];
 
+    d     = date.getDate()
+    m     = date.getMonth() + 1
+    yfull = date.getFullYear()
+    M     = date.getMinutes()
+    H     = date.getHours()
+
     // for less than 6 days
     // weekday HH::MM
     if (diff < (60 * 60 * 24 * 6))
       string = weekday + ' ' + date.getHours() + ':' + (date.getMinutes() < 10 ? '0':'') + date.getMinutes()
     else if (current.getYear() == date.getYear())
-       string = weekday + ' ' + date.getDate() + '. ' + month + ' ' + date.getHours() + ":" + (date.getMinutes() < 10 ? '0':'') + date.getMinutes()
+       string = weekday + ' ' + date.getDate() + '. ' + month + ' ' + (H < 10 ? '0':'') + H + ":" + (M < 10 ? '0':'') + M
     else
-       string = weekday + ' ' + date
+       string = weekday + ' ' + (m < 10 ? '0':'') + m + '/' + (d < 10 ? '0':'') + d + '/' + (yfull) + ' ' + (H < 10 ? '0':'') + H + ":" + (M < 10 ? '0':'') + M
     return string;
   }
 

script/build/database_config.rb

@@ -0,0 +1,38 @@
+require 'yaml'
+
+cnf = YAML.load_file(File.join(__dir__, '../../config/database/database.yml'))
+
+cnf.delete('default')
+
+hostsfile = '/etc/hosts'
+database  = %w[postgresql mysql].find do |possible_database|
+  File.foreach(hostsfile).any? { |l| l[possible_database] }
+end
+
+raise "Can't find any supported database in #{hostsfile}." if database.nil?
+
+db_settings_map = {
+  'postgresql' => {
+    'adapter'  => 'postgresql',
+    'username' => 'zammad',
+    'password' => 'zammad',
+    'host'     => 'postgresql', # db alias from gitlab-ci.yml
+  },
+  'mysql' => {
+    'adapter'  => 'mysql2',
+    'username' => 'root',
+    'password' => 'zammad',
+    'host'     => 'mysql', # db alias from gitlab-ci.yml
+  }
+}
+
+# fetch DB settings from settings map and fallback to postgresql
+db_settings = db_settings_map.fetch(database) { db_settings_map['postgresql'] }
+
+%w[development test production].each do |environment|
+  cnf[environment].merge!(db_settings)
+end
+
+File.open('config/database.yml', 'w') do |file|
+  file.write(Psych.dump(cnf))
+end

script/build/test_cleanup.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-rake db:drop RAILS_ENV=test
-rake db:drop RAILS_ENV=production

script/build/test_db_config.sh

@@ -1,74 +0,0 @@
-#!/bin/bash
-
-if [ "$RNAME" == '' ]; then
-  echo "ERROR: Env RNAME is missing!"
-  exit 1
-fi
-
-if [ "$RDB" == '' ]; then
-  echo "ERROR: Env RDB is missing!"
-  exit 1
-fi
-
-export RAND=`perl -e 'print int(rand(999999999))'`
-DBNAME="$RNAME-$RAND"
-DBFILE=config/database.yml
-DB_PRODUCTION=$DBNAME"_production"
-DB_TEST=$DBNAME"_test"
-DB_DEVELOPMENT=$DBNAME"_development"
-
-echo "Creating $DBFILE for $RDB tests with $DBNAME"
-
-if [ "$RDB" == 'mysql' ]; then
-  echo "production:" > $DBFILE
-  echo "  adapter: mysql2" >> $DBFILE
-  echo "  database: $DB_PRODUCTION" >> $DBFILE
-  echo "  pool: 50" >> $DBFILE
-  echo "  timeout: 5000" >> $DBFILE
-  echo "  encoding: utf8" >> $DBFILE
-  echo "  username: root" >> $DBFILE
-  echo "  password:" >> $DBFILE
-  echo "test:" >> $DBFILE
-  echo "  adapter: mysql2" >> $DBFILE
-  echo "  database: $DB_TEST" >> $DBFILE
-  echo "  pool: 50" >> $DBFILE
-  echo "  timeout: 5000" >> $DBFILE
-  echo "  encoding: utf8" >> $DBFILE
-  echo "  username: root" >> $DBFILE
-  echo "  password:" >> $DBFILE
-  echo "development:" >> $DBFILE
-  echo "  adapter: mysql2" >> $DBFILE
-  echo "  database: $DB_DEVELOPMENT" >> $DBFILE
-  echo "  pool: 50" >> $DBFILE
-  echo "  timeout: 5000" >> $DBFILE
-  echo "  encoding: utf8" >> $DBFILE
-  echo "  username: root" >> $DBFILE
-  echo "  password:" >> $DBFILE
-fi
-
-if [ "$RDB" == 'postgresql' ]; then
-  echo "production:" > $DBFILE
-  echo "  adapter: postgresql" >> $DBFILE
-  echo "  database: $DB_PRODUCTION" >> $DBFILE
-  echo "  pool: 50" >> $DBFILE
-  echo "  timeout: 5000" >> $DBFILE
-  echo "  encoding: utf8" >> $DBFILE
-  echo "  username: gitlab-runner" >> $DBFILE
-  echo "  password:" >> $DBFILE
-  echo "test:" >> $DBFILE
-  echo "  adapter: postgresql" >> $DBFILE
-  echo "  database: $DB_TEST" >> $DBFILE
-  echo "  pool: 50" >> $DBFILE
-  echo "  timeout: 5000" >> $DBFILE
-  echo "  encoding: utf8" >> $DBFILE
-  echo "  username: gitlab-runner" >> $DBFILE
-  echo "  password:" >> $DBFILE
-  echo "development:" >> $DBFILE
-  echo "  adapter: postgresql" >> $DBFILE
-  echo "  database: $DB_DEVELOPMENT" >> $DBFILE
-  echo "  pool: 50" >> $DBFILE
-  echo "  timeout: 5000" >> $DBFILE
-  echo "  encoding: utf8" >> $DBFILE
-  echo "  username: gitlab-runner" >> $DBFILE
-  echo "  password:" >> $DBFILE
-fi

script/build/test_shutdown.sh

@@ -1,36 +0,0 @@
-#!/bin/bash
-RAILS_ENV=$1
-APP_PORT=$2
-WS_PORT=$3
-EXIT=$4 || 0
-WITH_DB=$5 || 0
-WITH_ELASTICSEARCH=$6 || 0
-
-SERVER_PID='tmp/pids/server.pid'
-LOG_HOST='cilog@schneeberg.znuny.com'
-
-bundle exec script/scheduler.rb stop
-bundle exec script/websocket-server.rb stop
-kill $(cat $SERVER_PID)
-
-sleep 5
-if [ -f $SERVER_PID ]; then
-   kill -9 $(cat $SERVER_PID)
-fi
-
-if test $WITH_DB -eq 1; then
-  script/build/test_cleanup.sh
-fi
-
-if test $WITH_ELASTICSEARCH -eq 1; then
-  rake searchindex:drop
-fi
-
-# if build has failed, copy logs for analyzing
-if test $EXIT -eq 1; then
-  ssh $LOG_HOST "mkdir -p logs/$CI_BUILD_ID"
-  scp -C log/* $LOG_HOST:logs/$CI_BUILD_ID/
-  scp -C tmp/screenshot* $LOG_HOST:logs/$CI_BUILD_ID/
-fi
-
-exit $EXIT

script/build/test_startup.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-RAILS_ENV=$1
-APP_PORT=$2
-WS_PORT=$3
-WITH_DB=$4 || 0
-WITH_ELASTICSEARCH=$5 || 0
-
-if test $WITH_DB -eq 1; then
-  script/bootstrap.sh
-fi
-
-export ZAMMAD_SETTING_TTL=15
-
-rails r "Setting.set('developer_mode', true)"
-rails r "Setting.set('websocket_port', '$WS_PORT')"
-rails r "Setting.set('fqdn', '$IP:$BROWSER_PORT')"
-rails r "Setting.set('chat_agent_idle_timeout', '45')"
-
-if test $WITH_ELASTICSEARCH -eq 1; then
-  rails r "Setting.set('es_url', 'http://localhost:9200')"
-  rails r "Setting.set('es_index', 'browser_test_$CI_BUILD_ID')"
-  rake searchindex:rebuild
-fi
-
-echo "env used for script/build/test_startup.sh $1 $2 $3"
-echo "export RAILS_ENV=$RAILS_ENV"
-echo "export IP=$IP"
-echo "export BROWSER_PORT=$BROWSER_PORT"
-
-#rails s puma -d --pid tmp/pids/server.pid --bind 0.0.0.0 --port $APP_PORT
-bundle exec puma --pidfile tmp/pids/server.pid -d -p $APP_PORT -e $RAILS_ENV
-bundle exec script/websocket-server.rb start -d -p $WS_PORT
-bundle exec script/scheduler.rb start

spec/lib/core_ext/string_spec.rb

@@ -62,5 +62,42 @@ RSpec.describe String do
         end
       end
     end
+
+    context 'perforamnce' do
+      let(:subject) { original_string.encode(input_encoding) }
+
+      context 'with utf8_encode in iso-8859-1' do
+        let(:original_string) { 'äöü0' * 999_999 }
+        let(:input_encoding) { Encoding::ISO_8859_1 }
+
+        it 'detects the input encoding' do
+          Timeout.timeout(1) do
+            expect(subject.utf8_encode(from: 'iso-8859-1')).to eq(original_string)
+          end
+        end
+      end
+
+      context 'with utf8_encode in utf-8' do
+        let(:original_string) { 'äöü0' * 999_999 }
+        let(:input_encoding) { Encoding::UTF_8 }
+
+        it 'detects the input encoding' do
+          Timeout.timeout(1) do
+            expect(subject.utf8_encode(from: 'utf-8')).to eq(original_string)
+          end
+        end
+      end
+
+      context 'with utf8_encode in iso-8859-1 and charset detection' do
+        let(:original_string) { 'äöü0' * 199_999 }
+        let(:input_encoding) { Encoding::ISO_8859_1 }
+
+        it 'detects the input encoding' do
+          Timeout.timeout(8) do
+            expect(subject.utf8_encode(from: 'utf-8')).to eq(original_string)
+          end
+        end
+      end
+    end
   end
 end

spec/lib/sequencer/unit/import/zendesk/organization_fields_spec.rb

@@ -0,0 +1,6 @@
+require 'rails_helper'
+require 'lib/sequencer/unit/import/zendesk/sub_sequence/base_examples'
+
+RSpec.describe Sequencer::Unit::Import::Zendesk::OrganizationFields, sequencer: :unit do
+  include_examples 'Sequencer::Unit::Import::Zendesk::SubSequence::Base'
+end

spec/lib/sequencer/unit/import/zendesk/sub_sequence/base_examples.rb

@@ -0,0 +1,61 @@
+RSpec.shared_examples 'Sequencer::Unit::Import::Zendesk::SubSequence::Base' do
+  before do
+    allow(params[:client]).to receive(collection_name).and_return(client_collection)
+    allow(client_collection).to receive(:all!).and_raise(api_error)
+  end
+
+  let(:params) do
+    {
+      dry_run: false,
+      import_job: instance_double(ImportJob),
+      client: double('ZendeskAPI'),
+      group_map: {},         # required by Tickets
+      organization_map: {},  # required by Tickets
+      ticket_field_map: {},  # required by Tickets
+      user_map: {},          # required by Tickets
+    }
+  end
+
+  let(:collection_name) { described_class.name.demodulize.snakecase.to_sym }
+  let(:client_collection) { double('ZendeskAPI::Collection') }
+  let(:api_error) { ZendeskAPI::Error::NetworkError.new('Mock err msg', response_obj) }
+  let(:response_obj) { double('Faraday::Response') }
+
+  # https://github.com/zammad/zammad/issues/2262
+  context 'for lowest-tier Zendesk subscriptions ("Essential")' do
+    shared_examples 'Zendesk import data (only available on Team tier and up)' do
+      context 'when API returns 403 forbidden during sync' do
+        before { allow(response_obj).to receive(:status).and_return(403) }
+
+        it 'rescues the resulting exception' do
+          expect { process(params) }.not_to raise_error
+        end
+      end
+
+      context 'when API returns other errors' do
+        before { allow(response_obj).to receive(:status).and_return(500) }
+
+        # https://github.com/zammad/zammad/issues/2262
+        it 'does not rescue the resulting exception' do
+          expect { process(params) }.to raise_error(api_error)
+        end
+      end
+    end
+
+    shared_examples 'Zendesk import data (available on all tiers)' do
+      context 'if API returns 403 forbidden during sync' do
+        before { allow(response_obj).to receive(:status).and_return(403) }
+
+        it 'does not rescue the resulting exception' do
+          expect { process(params) }.to raise_error(api_error)
+        end
+      end
+    end
+
+    if described_class.name.demodulize.in?(%w[UserFields OrganizationFields])
+      include_examples 'Zendesk import data (only available on Team tier and up)'
+    else
+      include_examples 'Zendesk import data (available on all tiers)'
+    end
+  end
+end

spec/lib/sequencer/unit/import/zendesk/tickets_spec.rb

@@ -0,0 +1,6 @@
+require 'rails_helper'
+require 'lib/sequencer/unit/import/zendesk/sub_sequence/base_examples'
+
+RSpec.describe Sequencer::Unit::Import::Zendesk::Tickets, sequencer: :unit do
+  include_examples 'Sequencer::Unit::Import::Zendesk::SubSequence::Base'
+end

spec/lib/sequencer/unit/import/zendesk/user_fields_spec.rb

@@ -0,0 +1,6 @@
+require 'rails_helper'
+require 'lib/sequencer/unit/import/zendesk/sub_sequence/base_examples'
+
+RSpec.describe Sequencer::Unit::Import::Zendesk::UserFields, sequencer: :unit do
+  include_examples 'Sequencer::Unit::Import::Zendesk::SubSequence::Base'
+end

spec/models/concerns/can_lookup_examples.rb

@@ -4,8 +4,8 @@ RSpec.shared_examples 'CanLookup' do
   describe '::lookup' do
     let(:subject)            { described_class }
     let(:ensure_instance)    { create(subject.to_s.downcase) if subject.none? }
-    let(:string_attributes)  { (%i[name login email number] & subject.attribute_names.map(&:to_sym)) }
-    let(:non_attributes)     { (%i[id name login email number] - subject.attribute_names.map(&:to_sym)) }
+    let(:string_attributes)  { subject.lookup_keys }
+    let(:non_attributes)     { (subject.attribute_names.map(&:to_sym) - subject.lookup_keys) }
     let(:lookup_id)          { 1 }
     let(:cache_key)          { "#{subject}::#{lookup_id}" }
     let(:cache_expiry_param) { { expires_in: 7.days } }
@@ -35,7 +35,7 @@ RSpec.shared_examples 'CanLookup' do
     end
 
     it 'accepts exactly one attribute-value pair' do
-      expect { subject.lookup(:id => lookup_id, string_attributes.sample => 'foo') }
+      expect { subject.lookup(id: lookup_id, some_attribute: 'foo') }
         .to raise_error(ArgumentError)
     end
 
@@ -69,5 +69,10 @@ RSpec.shared_examples 'CanLookup' do
         .to have_received(:read)
         .with(cache_key)
     end
+
+    it 'verify lookup keys' do
+      expect(subject.lookup_keys).to match_array((%i[id name login email number] & subject.attribute_names.map(&:to_sym)))
+    end
+
   end
 end

spec/requests/api_auth_spec.rb

@@ -2,6 +2,17 @@ require 'rails_helper'
 
 RSpec.describe 'Api Auth', type: :request do
 
+  around(:each) do |example|
+    orig = ActionController::Base.allow_forgery_protection
+
+    begin
+      ActionController::Base.allow_forgery_protection = true
+      example.run
+    ensure
+      ActionController::Base.allow_forgery_protection = orig
+    end
+  end
+
   let(:admin_user) do
     create(:admin_user)
   end
@@ -369,7 +380,10 @@ RSpec.describe 'Api Auth', type: :request do
     it 'does session auth - admin' do
       create(:admin_user, login: 'api-admin@example.com', password: 'adminpw')
 
-      post '/api/v1/signin', params: { username: 'api-admin@example.com', password: 'adminpw', fingerprint: '123456789' }
+      get '/'
+      token = response.headers['CSRF-TOKEN']
+
+      post '/api/v1/signin', params: { username: 'api-admin@example.com', password: 'adminpw', fingerprint: '123456789' }, headers: { 'X-CSRF-Token' => token }
       expect(response.header.key?('Access-Control-Allow-Origin')).to be_falsey
       expect(response).to have_http_status(201)
 

spec/requests/calendar_spec.rb

@@ -55,7 +55,8 @@ RSpec.describe 'Calendars', type: :request do
       expect(json_response['ical_feeds']['http://www.google.com/calendar/ical/de.austrian%23holiday%40group.v.calendar.google.com/public/basic.ics']).to eq('Austria')
       expect(json_response['timezones']).to be_truthy
       expect(json_response['timezones']['Africa/Johannesburg']).to eq(2)
-      expect(json_response['timezones']['America/Sitka']).to eq(-8)
+      expect(json_response['timezones']['America/Sitka']).to be_between(-9, -8)
+      expect(json_response['timezones']['Europe/Berlin']).to be_between(1, 2)
       expect(json_response['assets']).to be_truthy
     end
   end

spec/requests/integration/cti_spec.rb

@@ -182,7 +182,7 @@ RSpec.describe 'Integration CTI', type: :request do
       expect(log.initialized_at).to be_truthy
       expect(log.start_at).to be_nil
       expect(log.end_at).to be_truthy
-      expect(log.duration_waiting_time).to eq(2)
+      expect(log.duration_waiting_time).to be_between(2, 3)
       expect(log.duration_talking_time).to be_nil
 
       # outbound - II - new call
@@ -226,7 +226,7 @@ RSpec.describe 'Integration CTI', type: :request do
       expect(log.initialized_at).to be_truthy
       expect(log.start_at).to be_truthy
       expect(log.end_at).to be_nil
-      expect(log.duration_waiting_time).to eq(2)
+      expect(log.duration_waiting_time).to be_between(2, 3)
       expect(log.duration_talking_time).to be_nil
 
       travel 2.seconds
@@ -249,8 +249,8 @@ RSpec.describe 'Integration CTI', type: :request do
       expect(log.initialized_at).to be_truthy
       expect(log.start_at).to be_truthy
       expect(log.end_at).to be_truthy
-      expect(log.duration_waiting_time).to eq(2)
-      expect(log.duration_talking_time).to eq(2)
+      expect(log.duration_waiting_time).to be_between(2, 3)
+      expect(log.duration_talking_time).to be_between(2, 3)
 
       # inbound - I - new call
       params = 'event=newCall&direction=in&to=4930600000000&from=4912347114711&call_id=1234567890-3&user%5B%5D=user+1'

spec/requests/long_polling_spec.rb

@@ -0,0 +1,124 @@
+require 'rails_helper'
+
+RSpec.describe 'LongPolling', type: :request do
+
+  let(:agent_user) do
+    create(:agent_user)
+  end
+
+  before(:each) do
+    Sessions.sessions.each do |client_id|
+      Sessions.destroy(client_id)
+    end
+    Sessions.spool_delete
+  end
+
+  describe 'request handling' do
+
+    it 'receive without client_id - no user login' do
+      get '/api/v1/message_receive', params: { data: {} }, as: :json
+      expect(response).to have_http_status(422)
+      expect(json_response).to be_a_kind_of(Hash)
+      expect(json_response['error']).to eq('Invalid client_id receive!')
+    end
+
+    it 'send without client_id - no user login' do
+      get '/api/v1/message_send', params: { data: {} }, as: :json
+      expect(response).to have_http_status(200)
+      expect(json_response).to be_a_kind_of(Hash)
+      expect(json_response['client_id'].to_i).to be_between(1, 9_999_999_999)
+
+      client_id = json_response['client_id']
+      get '/api/v1/message_send', params: { client_id: client_id, data: { event: 'spool' } }, as: :json
+      expect(response).to have_http_status(200)
+      expect(json_response).to be_a_kind_of(Hash)
+      expect(json_response['client_id'].to_i).to be_between(1, 9_999_999_999)
+
+      get '/api/v1/message_receive', params: { client_id: client_id, data: {} }, as: :json
+      expect(response).to have_http_status(422)
+      expect(json_response).to be_a_kind_of(Hash)
+      expect(json_response['error']).to eq('Invalid client_id receive!')
+    end
+
+    it 'receive without client_id' do
+      authenticated_as(agent_user)
+      get '/api/v1/message_receive', params: { data: {} }, as: :json
+      expect(response).to have_http_status(422)
+      expect(json_response).to be_a_kind_of(Hash)
+      expect(json_response['error']).to eq('Invalid client_id receive!')
+    end
+
+    it 'receive without wrong client_id' do
+      authenticated_as(agent_user)
+      get '/api/v1/message_receive', params: { client_id: 'not existing', data: {} }, as: :json
+      expect(response).to have_http_status(422)
+      expect(json_response).to be_a_kind_of(Hash)
+      expect(json_response['error']).to eq('Invalid client_id receive!')
+    end
+
+    it 'send without client_id' do
+      authenticated_as(agent_user)
+      get '/api/v1/message_send', params: { data: {} }, as: :json
+      expect(response).to have_http_status(200)
+      expect(json_response).to be_a_kind_of(Hash)
+      expect(json_response['client_id'].to_i).to be_between(1, 9_999_999_999)
+    end
+
+    it 'send with client_id' do
+      Sessions.create('123456', {}, { type: 'ajax' })
+      authenticated_as(agent_user)
+      get '/api/v1/message_send', params: { client_id: '123456', data: {} }, as: :json
+      expect(response).to have_http_status(200)
+      expect(json_response).to be_a_kind_of(Hash)
+      expect(json_response).to eq({})
+    end
+
+    it 'send event spool and receive data' do
+
+      authenticated_as(agent_user)
+      get '/api/v1/message_send', params: { data: { event: 'login' } }, as: :json
+      expect(response).to have_http_status(200)
+      expect(json_response['client_id'].to_i).to be_between(1, 9_999_999_999)
+      client_id = json_response['client_id']
+
+      get '/api/v1/message_receive', params: { client_id: client_id, data: {} }, as: :json
+      expect(response).to have_http_status(200)
+      expect(json_response).to be_a_kind_of(Array)
+      expect(json_response).to eq([{ 'data' => { 'success' => true }, 'event' => 'ws:login' }])
+
+      get '/api/v1/message_send', params: { client_id: client_id, data: { event: 'spool' } }, as: :json
+      expect(response).to have_http_status(200)
+      expect(json_response).to be_a_kind_of(Hash)
+      expect(json_response).to eq({})
+
+      get '/api/v1/message_receive', params: { client_id: client_id, data: {} }, as: :json
+      expect(response).to have_http_status(200)
+      expect(json_response).to be_a_kind_of(Array)
+      expect(json_response[0]['event']).to eq('spool:sent')
+      expect(json_response[0]['event']).to eq('spool:sent')
+      expect(json_response.count).to eq(1)
+
+      spool_list = Sessions.spool_list(Time.now.utc.to_i, agent_user.id)
+      expect(spool_list).to eq([])
+
+      get '/api/v1/message_send', params: { client_id: client_id, data: { event: 'broadcast', spool: true, recipient: { user_id: [agent_user.id] }, data: { taskbar_id: 9_391_633 } } }, as: :json
+      expect(response).to have_http_status(200)
+      expect(json_response).to be_a_kind_of(Hash)
+      expect(json_response).to eq({})
+
+      get '/api/v1/message_receive', params: { client_id: client_id, data: {} }, as: :json
+      expect(response).to have_http_status(200)
+      expect(json_response).to be_a_kind_of(Hash)
+      expect(json_response).to eq({ 'event' => 'pong' })
+
+      travel 2.seconds
+
+      spool_list = Sessions.spool_list(Time.now.utc.to_i, agent_user.id)
+      expect(spool_list).to eq([])
+
+      spool_list = Sessions.spool_list(nil, agent_user.id)
+      expect(spool_list).to eq([{ message: { 'taskbar_id' => 9_391_633 }, type: 'direct' }])
+    end
+
+  end
+end

spec/requests/organization_spec.rb

@@ -512,8 +512,8 @@ RSpec.describe 'Organization', type: :request, searchindex: true do
       expect(json_response['records'].count).to eq(2)
       expect(json_response['result']).to eq('failed')
       expect(json_response['errors'].count).to eq(2)
-      expect(json_response['errors'][0]).to eq("Line 1: unknown attribute 'name2' for Organization.")
-      expect(json_response['errors'][1]).to eq("Line 2: unknown attribute 'name2' for Organization.")
+      expect(json_response['errors'][0]).to eq("Line 1: Unable to create record - unknown attribute 'name2' for Organization.")
+      expect(json_response['errors'][1]).to eq("Line 2: Unable to create record - unknown attribute 'name2' for Organization.")
 
       # valid file try
       csv_file_path = Rails.root.join('test', 'data', 'csv', 'organization_simple.csv')

spec/requests/text_module_spec.rb

@@ -59,8 +59,8 @@ RSpec.describe 'Text Module', type: :request do
       expect(json_response['records'].count).to eq(2)
       expect(json_response['result']).to eq('failed')
       expect(json_response['errors'].count).to eq(2)
-      expect(json_response['errors'][0]).to eq("Line 1: unknown attribute 'keywords2' for TextModule.")
-      expect(json_response['errors'][1]).to eq("Line 2: unknown attribute 'keywords2' for TextModule.")
+      expect(json_response['errors'][0]).to eq("Line 1: Unable to create record - unknown attribute 'keywords2' for TextModule.")
+      expect(json_response['errors'][1]).to eq("Line 2: Unable to create record - unknown attribute 'keywords2' for TextModule.")
 
       # valid file try
       csv_file_path = Rails.root.join('test', 'data', 'csv', 'text_module_simple.csv')

spec/requests/ticket_spec.rb

@@ -2079,4 +2079,37 @@ RSpec.describe 'Ticket', type: :request do
       expect(json_response['tickets']).to eq([ticket2.id, ticket1.id])
     end
   end
+
+  describe 'stats' do
+    let(:ticket1) { create(:ticket, customer: customer, organization: organization) }
+    let(:ticket2) { create(:ticket, customer: customer, organization: organization) }
+    let(:ticket3) { create(:ticket, customer: customer, organization: organization) }
+    let(:customer) { create(:customer_user, organization: organization) }
+    let(:organization) { create(:organization, shared: false) }
+
+    before do
+      authenticated_as(admin_user)
+
+      ticket1
+      travel 2.minutes
+      ticket2
+      travel 2.minutes
+      ticket3
+      travel 2.minutes
+      ticket2.touch # rubocop:disable Rails/SkipsModelValidations
+    end
+
+    # https://github.com/zammad/zammad/issues/2296
+    it 'orders tickets by created_at desc (#2296)' do
+      get '/api/v1/ticket_stats', params: { organization_id: organization.id, user_id: customer.id }, as: :json
+
+      expect(response).to have_http_status(200)
+      expect(json_response)
+        .to be_a_kind_of(Hash)
+        .and include('user' => hash_including('open_ids' => [ticket3.id, ticket2.id, ticket1.id]))
+        .and include('organization' => hash_including('open_ids' => [ticket3.id, ticket2.id, ticket1.id]))
+    end
+
+  end
+
 end

spec/requests/user_spec.rb

@@ -872,8 +872,8 @@ RSpec.describe 'User', type: :request, searchindex: true do
       expect(json_response['records'].count).to eq(2)
       expect(json_response['result']).to eq('failed')
       expect(json_response['errors'].count).to eq(2)
-      expect(json_response['errors'][0]).to eq("Line 1: unknown attribute 'firstname2' for User.")
-      expect(json_response['errors'][1]).to eq("Line 2: unknown attribute 'firstname2' for User.")
+      expect(json_response['errors'][0]).to eq("Line 1: Unable to create record - unknown attribute 'firstname2' for User.")
+      expect(json_response['errors'][1]).to eq("Line 2: Unable to create record - unknown attribute 'firstname2' for User.")
 
       # valid file try
       csv_file_path = Rails.root.join('test', 'data', 'csv', 'user_simple.csv')

test/browser/admin_calendar_sla_test.rb

@@ -13,7 +13,7 @@ class AdminCalendarSlaTest < TestCase
     calendar_name = "ZZZ some calendar #{rand(99_999_999)}"
     sla_name = "ZZZ some sla #{rand(99_999_999)}"
     timezone = 'Europe/Berlin'
-    timezone_verify = 'Europe/Berlin (GMT+2)'
+    timezone_verify = "Europe/Berlin\s\\(GMT\\+(2|1)\\)"
     calendar_create(
       data: {
         name:     calendar_name,
@@ -66,7 +66,7 @@ class AdminCalendarSlaTest < TestCase
     )
     watch_for(
       css: '.content.active .modal input.js-input',
-      value: Regexp.quote(timezone_verify),
+      value: timezone_verify,
       timeout: 4,
     )
     modal_close()

test/browser/admin_channel_email_test.rb

@@ -98,13 +98,7 @@ class AdminChannelEmailTest < TestCase
       value: 'Users',
     )
     click(css: '.modal button.js-submit')
-    sleep 2
-
-    watch_for_disappear(
-      css: '.modal',
-    )
-    sleep 2
-    exists_not(css: '.modal')
+    modal_disappear(timeout: 20)
 
     watch_for(
       css: '.content.active',

test/browser/admin_permissions_granular_vs_full_test.rb

@@ -11,7 +11,7 @@ class AdminPermissionsGranularVsFullTest < TestCase
     )
     tasks_close_all()
 
-    click(css: '.user-menu a[title=Admin')
+    click(css: 'a[href="#manage"]')
     click(css: '.content.active a[href="#manage/groups"]')
     click(css: '.content.active a[data-type="new"]')
 
@@ -21,26 +21,34 @@ class AdminPermissionsGranularVsFullTest < TestCase
     element.clear
     element.send_keys(new_group_name)
     click(css: '.modal button.js-submit')
-
-    sleep(1)
+    modal_disappear
 
     click(css: '.content.active a[href="#manage/users"]')
 
     user_css = '.user-list .js-tableBody tr td'
     watch_for(css: user_css)
-    click(css: user_css)
+    @browser.find_elements(css: '.content.active .user-list td:first-child').each do |entry|
+      next if entry.text.strip != 'master@example.com'
+
+      entry.click
+      break
+    end
 
     modal_ready
 
     scroll_script = "var el = document.getElementsByClassName('modal')[0];"
     scroll_script += 'el.scrollTo(0, el.scrollHeight);'
-
     @browser.execute_script scroll_script
 
     group = @browser.find_elements(css: '.modal .settings-list tbody tr').find do |el|
       el.find_element(css: 'td').text == new_group_name
     end
 
+    if !group
+      screenshot(comment: 'group_not_found')
+      raise "Can't find group #{new_group_name}"
+    end
+
     toggle_checkbox(group, 'full')
     sleep(1)
     assert(checkbox_is_selected(group, 'full'))

test/browser/admin_role_test.rb

@@ -226,4 +226,34 @@ class AdminRoleTest < TestCase
     exists_not(css: '[href="#manage"]')
 
   end
+
+  # regression test for issue #2332 - Role-Filter shows inactive Roles
+  def test_inactive_roles_do_not_show_in_role_filter
+    name = "some role #{rand(99_999_999)}"
+
+    @browser = browser_instance
+    login(
+      username: 'master@example.com',
+      password: 'test',
+      url: browser_url,
+    )
+    tasks_close_all()
+
+    role_create(
+      data: {
+        name:   name,
+        active: false
+      }
+    )
+
+    click(
+      css:  '.content.active a[href="#manage/users"]',
+    )
+
+    # an inactive role should not appear in the role filter tabs
+    match_not(
+      css:   '.content.active .userSearch',
+      value: name,
+    )
+  end
 end

test/browser/agent_ticket_tag_test.rb

@@ -353,7 +353,7 @@ class AgentTicketTagTest < TestCase
   end
 
   def test_b_tags
-    tag_prefix = "tag#{rand(999_999_999)}"
+    tag_prefix = "tag#{rand(1000)}"
 
     @browser = browser_instance
     login(

test/browser/maintenance_login_message_test.rb

@@ -63,6 +63,7 @@ class MaintenanceLoginMessageTest < TestCase
     watch_for_disappear(
       browser: browser2,
       css: '.js-maintenanceLogin',
+      timeout: 30,
     )
   end
 

test/browser_test_helper.rb

@@ -46,7 +46,15 @@ class TestCase < Test::Unit::TestCase
   end
 
   def browser_url
-    ENV['BROWSER_URL'] || 'http://localhost:3000'
+    return ENV['BROWSER_URL'] if ENV['BROWSER_URL'].present?
+
+    "http://#{host}:3000"
+  end
+
+  def host
+    return 'localhost' if ENV['CI'].blank?
+
+    Socket.ip_address_list.detect(&:ipv4_private?).ip_address
   end
 
   def browser_instance
@@ -75,10 +83,10 @@ class TestCase < Test::Unit::TestCase
       begin
         local_browser = browser_instance_remote
         break
-      rescue
+      rescue => e
         wait_until_ready = rand(5..13)
+        log('browser_instance', { rescure: true, count: count, sleep: wait_until_ready, exception: e })
         sleep wait_until_ready
-        log('browser_instance', { rescure: true, count: count, sleep: wait_until_ready })
       end
     end
 
@@ -93,10 +101,22 @@ class TestCase < Test::Unit::TestCase
     if ENV['BROWSER_VERSION']
       caps.version  = ENV['BROWSER_VERSION']
     end
+
+    # (ironically) required for timeout checks
+    # https://github.com/zalando/zalenium/issues/469#issuecomment-371417340
+    # https://opensource.zalando.com/zalenium/#usage
+    caps['idleTimeout'] = 300
+
+    http_client = Selenium::WebDriver::Remote::Http::Default.new(
+      open_timeout: 120,
+      read_timeout: 120
+    )
+
     local_browser = Selenium::WebDriver.for(
       :remote,
       url: ENV['REMOTE_URL'],
       desired_capabilities: caps,
+      http_client: http_client,
     )
     @browsers[local_browser.hash] = local_browser
     browser_instance_preferences(local_browser)
@@ -3712,6 +3732,10 @@ wait untill text in selector disabppears
       end
     end
 
+    if data[:active] == false
+      select(css: 'select[name="active"]', value: 'inactive')
+    end
+
     instance.find_elements(css: '.modal button.js-submit')[0].click
     modal_disappear(browser: instance)
     11.times do

test/data/string/html2text1.html

@@ -0,0 +1,20 @@
+<html>
+<title>some title</title>
+<body>
+
+<div>hello</div>
+
+<p>some word <a href="http://example.com?domain?example.com">some url</a> and the end.</p>
+<p>some word <a href="http://example.com?domain?example.com">some url</a> and the end.</p>
+<p>some word <a href="http://example.com?domain?example.com">some url</a> and the end.</p>
+<p>some word <a href="http://example.com?domain?example.com">some url</a> and the end.</p>
+<p>some word <a href="http://example.com?domain?example.com">some url</a> and the end.</p>
+<p>some word <a href="http://example.com?domain?example.com">some url</a> and the end.</p>
+<p>some word <a href="http://example.com?domain?example.com">some url</a> and the end.</p>
+<p>some word <a href="http://example.com?domain?example.com">some url</a> and the end.</p>
+<p>some word <a href="http://example.com?domain?example.com">some url</a> and the end.</p>
+<p>some word <a href="http://example.com?domain?example.com">some url</a> and the end.</p>
+<p>some word <a href="http://example.com?domain?example.com">some url</a> and the end.</p>
+
+</body>
+</html>

test/data/string/html2text1.txt

@@ -0,0 +1,26 @@
+some title
+hello
+
+some word [1] some url and the end.
+some word [2] some url and the end.
+some word [3] some url and the end.
+some word [4] some url and the end.
+some word [5] some url and the end.
+some word [6] some url and the end.
+some word [7] some url and the end.
+some word [8] some url and the end.
+some word [9] some url and the end.
+some word [10] some url and the end.
+some word [11] some url and the end.
+
+[1] http://example.com?domain?example.com
+[2] http://example.com?domain?example.com
+[3] http://example.com?domain?example.com
+[4] http://example.com?domain?example.com
+[5] http://example.com?domain?example.com
+[6] http://example.com?domain?example.com
+[7] http://example.com?domain?example.com
+[8] http://example.com?domain?example.com
+[9] http://example.com?domain?example.com
+[10] http://example.com?domain?example.com
+[11] http://example.com?domain?example.com

test/unit/aaa_string_test.rb

@@ -56,64 +56,64 @@ class AaaStringTest < ActiveSupport::TestCase
     modul  = 'test'
     result = 'test'
     modul.to_filename
-    assert_equal(result,  modul)
+    assert_equal(result, modul)
 
     modul  = 'Some::File'
     result = 'Some::File'
     modul.to_filename
-    assert_equal(result,  modul)
+    assert_equal(result, modul)
   end
 
   test 'to_filename function' do
     modul  = 'test'
     result = 'test'
-    assert_equal(result,  modul.to_filename)
+    assert_equal(result, modul.to_filename)
 
     modul  = 'Some::File'
     result = 'some/file'
-    assert_equal(result,  modul.to_filename)
+    assert_equal(result, modul.to_filename)
   end
 
   test 'to_classname ref' do
     modul  = 'test'
     result = 'test'
     modul.to_filename
-    assert_equal(result,  modul)
+    assert_equal(result, modul)
 
     modul  = 'some/file'
     result = 'some/file'
     modul.to_filename
-    assert_equal(result,  modul)
+    assert_equal(result, modul)
   end
 
   test 'to_classname function' do
     modul  = 'test'
     result = 'Test'
-    assert_equal(result,  modul.to_classname)
+    assert_equal(result, modul.to_classname)
 
     modul  = 'some/file'
     result = 'Some::File'
-    assert_equal(result,  modul.to_classname)
+    assert_equal(result, modul.to_classname)
 
     modul  = 'some/files'
     result = 'Some::Files'
-    assert_equal(result,  modul.to_classname)
+    assert_equal(result, modul.to_classname)
 
     modul  = 'some_test/files'
     result = 'SomeTest::Files'
-    assert_equal(result,  modul.to_classname)
+    assert_equal(result, modul.to_classname)
   end
 
   test 'html2text ref' do
     html   = 'test'
     result = 'test'
     html.html2text
-    assert_equal(result,  html)
+    assert_equal(result, html)
 
     html   = '<div>test</div>'
     result = '<div>test</div>'
     html.html2text
-    assert_equal(result,  html)
+    assert_equal(result, html)
   end
 
   test 'html2text function' do
@@ -458,6 +458,17 @@ Well as though adam took out here. Melvin will be more money. Called him into th
 Men-----------------------'
     assert_equal(result, html.html2text)
 
+    Timeout::timeout(2) do
+      html   = File.read(Rails.root.join('test', 'data', 'string', 'html2text1.html'))
+      result = File.read(Rails.root.join('test', 'data', 'string', 'html2text1.txt'))
+      assert_equal(result, html.html2text)
+    end
+
+    Timeout::timeout(2) do
+      html   = File.read(Rails.root.join('test', 'data', 'string', 'html2text2.html'))
+      result = File.read(Rails.root.join('test', 'data', 'string', 'html2text2.txt'))
+      assert_equal(result, html.html2text)
+    end
   end
 
   test 'html2html_strict function' do

test/unit/chat_test.rb

@@ -39,6 +39,91 @@ class ChatTest < ActiveSupport::TestCase
     Setting.set('chat', false)
   end
 
+  test 'instance_variable test' do
+    assert_nil(Sessions::Event::Base.instance_variable_get(:@database_connection))
+    assert_equal(Sessions::Event::ChatBase.instance_variable_get(:@database_connection), true)
+    assert_equal(Sessions::Event::ChatStatusAgent.instance_variable_get(:@database_connection), true)
+  end
+
+  # check if db connection is available for chat events
+  # see: https://github.com/zammad/zammad/issues/2353
+  test 'chat event db connection test' do
+
+    class DummyWs
+      def send(msg)
+        Rails.logger.info "WS send: #{msg}"
+      end
+    end
+
+    # with websockets
+    assert(User.first)
+
+    message = Sessions::Event.run(
+      event: 'login',
+      payload: {},
+      session: 123,
+      remote_ip: '127.0.0.1',
+      client_id: '123',
+      clients: {
+        '123' => {
+          websocket: DummyWs.new # to simulate a ws connection
+        }
+      },
+      options: {},
+    )
+    assert_equal(message, false)
+
+    assert_raises(ActiveRecord::ConnectionNotEstablished) do
+      User.first
+    end
+
+    message = Sessions::Event.run(
+      event: 'chat_status_customer',
+      payload: {},
+      session: 123,
+      remote_ip: '127.0.0.1',
+      client_id: '123',
+      clients: {
+        '123' => DummyWs.new # to simulate a ws connection
+      },
+      options: {},
+    )
+    assert_equal(message[:event], 'chat_error')
+
+    assert_raises(ActiveRecord::ConnectionNotEstablished) do
+      User.first
+    end
+
+    # re-establish connection
+    ActiveRecord::Base.establish_connection
+
+    # with ajax long polling
+    assert(User.first)
+    message = Sessions::Event.run(
+      event: 'login',
+      payload: {},
+      session: 123,
+      remote_ip: '127.0.0.1',
+      client_id: '123',
+      clients: {},
+      options: {},
+    )
+    assert_equal(message, false)
+    assert(User.first)
+
+    message = Sessions::Event.run(
+      event: 'chat_status_customer',
+      payload: {},
+      session: 123,
+      remote_ip: '127.0.0.1',
+      client_id: '123',
+      clients: {},
+      options: {},
+    )
+    assert_equal(message[:event], 'chat_error')
+    assert(User.first)
+  end
+
   test 'default test' do
 
     chat = Chat.create_or_update(

test/unit/organization_csv_import_test.rb

@@ -45,6 +45,20 @@ class OrganizationCsvImportTest < ActiveSupport::TestCase
     assert_equal('No records found in file/string for Organization.', result[:errors][0])
   end
 
+  test 'verify required lookup headers' do
+    csv_string = "firstname;lastname;active;\nfirstname-simple-import1;lastname-simple-import1;;true\nfirstname-simple-import2;lastname-simple-import2;false\n"
+    result = Organization.csv_import(
+      string: csv_string,
+      parse_params: {
+        col_sep: ';',
+      },
+      try: true,
+    )
+    assert_equal(true, result[:try])
+    assert_equal('failed', result[:result])
+    assert_equal('No lookup column like id,name for Organization found.', result[:errors][0])
+  end
+
   test 'simple import' do
 
     csv_string = "id;name;shared;domain;domain_assignment;active;note\n;org-simple-import1;true;org-simple-import1.example.com;false;true;some note1\n;org-simple-import2;true;org-simple-import2.example.com;false;false;some note2\n"
@@ -210,8 +224,8 @@ class OrganizationCsvImportTest < ActiveSupport::TestCase
     assert_equal(true, result[:try])
     assert_equal(2, result[:errors].count)
     assert_equal('failed', result[:result])
-    assert_equal("Line 1: unknown attribute 'not existing' for Organization.", result[:errors][0])
-    assert_equal("Line 2: unknown attribute 'not existing' for Organization.", result[:errors][1])
+    assert_equal("Line 1: Unable to create record - unknown attribute 'not existing' for Organization.", result[:errors][0])
+    assert_equal("Line 2: Unable to create record - unknown attribute 'not existing' for Organization.", result[:errors][1])
 
     assert_nil(Organization.find_by(name: 'organization-invalid-import1'))
     assert_nil(Organization.find_by(name: 'organization-invalid-import2'))
@@ -226,8 +240,8 @@ class OrganizationCsvImportTest < ActiveSupport::TestCase
     assert_equal(false, result[:try])
     assert_equal(2, result[:errors].count)
     assert_equal('failed', result[:result])
-    assert_equal("Line 1: unknown attribute 'not existing' for Organization.", result[:errors][0])
-    assert_equal("Line 2: unknown attribute 'not existing' for Organization.", result[:errors][1])
+    assert_equal("Line 1: Unable to create record - unknown attribute 'not existing' for Organization.", result[:errors][0])
+    assert_equal("Line 2: Unable to create record - unknown attribute 'not existing' for Organization.", result[:errors][1])
 
     assert_nil(Organization.find_by(name: 'organization-invalid-import1'))
     assert_nil(Organization.find_by(name: 'organization-invalid-import2'))

test/unit/text_module_csv_import_test.rb

@@ -49,6 +49,20 @@ class TextModuleCsvImportTest < ActiveSupport::TestCase
     assert_equal('No records found in file/string for TextModule.', result[:errors][0])
   end
 
+  test 'verify required lookup headers' do
+    csv_string = "firstname;lastname;active;\nfirstname-simple-import1;lastname-simple-import1;;true\nfirstname-simple-import2;lastname-simple-import2;false\n"
+    result = TextModule.csv_import(
+      string: csv_string,
+      parse_params: {
+        col_sep: ';',
+      },
+      try: true,
+    )
+    assert_equal(true, result[:try])
+    assert_equal('failed', result[:result])
+    assert_equal('No lookup column like id,name for TextModule found.', result[:errors][0])
+  end
+
   test 'simple import' do
     TextModule.create!(
       name: 'nsome name1',

test/unit/ticket_csv_import_test.rb

@@ -49,6 +49,20 @@ class TicketCsvImportTest < ActiveSupport::TestCase
     assert_equal('No records found in file/string for Ticket.', result[:errors][0])
   end
 
+  test 'verify required lookup headers' do
+    csv_string = "firstname;lastname;active;\nfirstname-simple-import1;lastname-simple-import1;;true\nfirstname-simple-import2;lastname-simple-import2;false\n"
+    result = Ticket.csv_import(
+      string: csv_string,
+      parse_params: {
+        col_sep: ';',
+      },
+      try: true,
+    )
+    assert_equal(true, result[:try])
+    assert_equal('failed', result[:result])
+    assert_equal('No lookup column like id,number for Ticket found.', result[:errors][0])
+  end
+
   test 'simple import' do
 
     csv_string = "id;number;title;state;priority;owner;customer;group;note\n;123456;some title1;new;2 normal;-;nicole.braun@zammad.org;Users;some note1\n;123457;some title2;closed;1 low;admin@example.com;nicole.braun@zammad.org;Users;some note2\n"
@@ -173,8 +187,8 @@ class TicketCsvImportTest < ActiveSupport::TestCase
     assert_equal(true, result[:try])
     assert_equal(2, result[:errors].count)
     assert_equal('failed', result[:result])
-    assert_equal("Line 1: unknown attribute 'not_existing' for Ticket.", result[:errors][0])
-    assert_equal("Line 2: unknown attribute 'not_existing' for Ticket.", result[:errors][1])
+    assert_equal("Line 1: Unable to create record - unknown attribute 'not_existing' for Ticket.", result[:errors][0])
+    assert_equal("Line 2: Unable to create record - unknown attribute 'not_existing' for Ticket.", result[:errors][1])
 
     assert_nil(Ticket.find_by(number: '123456'))
     assert_nil(Ticket.find_by(number: '123457'))
@@ -189,8 +203,8 @@ class TicketCsvImportTest < ActiveSupport::TestCase
     assert_equal(false, result[:try])
     assert_equal(2, result[:errors].count)
     assert_equal('failed', result[:result])
-    assert_equal("Line 1: unknown attribute 'not_existing' for Ticket.", result[:errors][0])
-    assert_equal("Line 2: unknown attribute 'not_existing' for Ticket.", result[:errors][1])
+    assert_equal("Line 1: Unable to create record - unknown attribute 'not_existing' for Ticket.", result[:errors][0])
+    assert_equal("Line 2: Unable to create record - unknown attribute 'not_existing' for Ticket.", result[:errors][1])
 
     assert_nil(Ticket.find_by(number: '123456'))
     assert_nil(Ticket.find_by(number: '123457'))

test/unit/user_csv_import_test.rb

@@ -44,9 +44,24 @@ class UserCsvImportTest < ActiveSupport::TestCase
     assert_equal('No records found in file/string for User.', result[:errors][0])
   end
 
+  test 'verify required lookup headers' do
+    csv_string = "firstname;lastname;active;\nfirstname-simple-import1;lastname-simple-import1;;true\nfirstname-simple-import2;lastname-simple-import2;false\n"
+    result = User.csv_import(
+      string: csv_string,
+      parse_params: {
+        col_sep: ';',
+      },
+      try: true,
+    )
+    assert_equal(true, result[:try])
+    assert_equal('failed', result[:result])
+    assert_equal('No lookup column like id,login,email for User found.', result[:errors][0])
+  end
+
   test 'simple import' do
 
-    csv_string = "login;firstname;lastname;email;active;\nuser-simple-import1;firstname-simple-import1;lastname-simple-import1;user-simple-import1@example.com;true\nuser-simple-import2;firstname-simple-import2;lastname-simple-import2;user-simple-import2@example.com;false\n"
+    count = User.count
+    csv_string = "login;firstname;lastname;email;active;\nuser-simple-IMPORT1;firstname-simple-import1;lastname-simple-import1;user-simple-IMPORT1@example.com ;true\nuser-simple-import2;firstname-simple-import2;lastname-simple-import2;user-simple-import2@example.com;false\n"
     result = User.csv_import(
       string: csv_string,
       parse_params: {
@@ -57,6 +72,9 @@ class UserCsvImportTest < ActiveSupport::TestCase
     assert_equal(true, result[:try])
     assert_equal(2, result[:records].count)
     assert_equal('success', result[:result])
+    assert_equal(2, result[:stats][:created])
+    assert_equal(0, result[:stats][:updated])
+    assert_equal(count, User.count)
 
     assert_nil(User.find_by(login: 'user-simple-import1'))
     assert_nil(User.find_by(login: 'user-simple-import2'))
@@ -71,6 +89,9 @@ class UserCsvImportTest < ActiveSupport::TestCase
     assert_equal(false, result[:try])
     assert_equal(2, result[:records].count)
     assert_equal('success', result[:result])
+    assert_equal(2, result[:stats][:created])
+    assert_equal(0, result[:stats][:updated])
+    assert_equal(count + 2, User.count)
 
     user1 = User.find_by(login: 'user-simple-import1')
     assert(user1)
@@ -87,6 +108,76 @@ class UserCsvImportTest < ActiveSupport::TestCase
     assert_equal(user2.email, 'user-simple-import2@example.com')
     assert_equal(user2.active, false)
 
+    result = User.csv_import(
+      string: csv_string,
+      parse_params: {
+        col_sep: ';',
+      },
+      try: false,
+    )
+    assert_equal(false, result[:try])
+    assert_equal(2, result[:records].count)
+    assert_equal('success', result[:result])
+    assert_equal(0, result[:stats][:created])
+    assert_equal(2, result[:stats][:updated])
+    assert_equal(count + 2, User.count)
+
+    user1_1 = user1
+    user1 = User.find_by(login: 'user-simple-import1')
+    assert(user1)
+    assert_equal(user1.login, 'user-simple-import1')
+    assert_equal(user1.firstname, 'firstname-simple-import1')
+    assert_equal(user1.lastname, 'lastname-simple-import1')
+    assert_equal(user1.email, 'user-simple-import1@example.com')
+    assert_equal(user1.active, true)
+    assert_equal(user1.updated_at, user1_1.updated_at)
+    user2 = user2
+    user2_1 = User.find_by(login: 'user-simple-import2')
+    assert(user2)
+    assert_equal(user2.login, 'user-simple-import2')
+    assert_equal(user2.firstname, 'firstname-simple-import2')
+    assert_equal(user2.lastname, 'lastname-simple-import2')
+    assert_equal(user2.email, 'user-simple-import2@example.com')
+    assert_equal(user2.active, false)
+    assert_equal(user2.updated_at, user2_1.updated_at)
+
+    travel 2.seconds
+
+    csv_string = "login;firstname;lastname;email;active;\n ;firstname-simple-import1;lastname-simple-import1;user-simple-IMPORT1@example.com ;true\n   user-simple-import2\t;firstname-simple-import2;lastname-simple-import2;;false\n"
+
+    result = User.csv_import(
+      string: csv_string,
+      parse_params: {
+        col_sep: ';',
+      },
+      try: false,
+    )
+    assert_equal(false, result[:try])
+    assert_equal(2, result[:records].count)
+    assert_equal(0, result[:stats][:created])
+    assert_equal(2, result[:stats][:updated])
+    assert_equal('success', result[:result])
+    assert_equal(count + 2, User.count)
+
+    user1_1 = user1
+    user1 = User.find_by(email: 'user-simple-import1@example.com')
+    assert(user1)
+    assert_equal(user1.login, 'user-simple-import1@example.com')
+    assert_equal(user1.firstname, 'firstname-simple-import1')
+    assert_equal(user1.lastname, 'lastname-simple-import1')
+    assert_equal(user1.email, 'user-simple-import1@example.com')
+    assert_equal(user1.active, true)
+    assert_not_equal(user1.updated_at, user1_1.updated_at)
+    user2 = user2
+    user2_1 = User.find_by(login: 'user-simple-import2')
+    assert(user2)
+    assert_equal(user2.login, 'user-simple-import2')
+    assert_equal(user2.firstname, 'firstname-simple-import2')
+    assert_equal(user2.lastname, 'lastname-simple-import2')
+    assert_equal(user2.email, 'user-simple-import2@example.com')
+    assert_equal(user2.active, false)
+    assert_equal(user2.updated_at, user2_1.updated_at)
+
     user1.destroy!
     user2.destroy!
   end
@@ -340,8 +431,8 @@ class UserCsvImportTest < ActiveSupport::TestCase
     assert_equal(true, result[:try])
     assert_equal(2, result[:errors].count)
     assert_equal('failed', result[:result])
-    assert_equal("Line 1: unknown attribute 'firstname2' for User.", result[:errors][0])
-    assert_equal("Line 2: unknown attribute 'firstname2' for User.", result[:errors][1])
+    assert_equal("Line 1: Unable to create record - unknown attribute 'firstname2' for User.", result[:errors][0])
+    assert_equal("Line 2: Unable to create record - unknown attribute 'firstname2' for User.", result[:errors][1])
 
     assert_nil(User.find_by(login: 'user-invalid-import1'))
     assert_nil(User.find_by(login: 'user-invalid-import2'))
@@ -356,8 +447,8 @@ class UserCsvImportTest < ActiveSupport::TestCase
     assert_equal(false, result[:try])
     assert_equal(2, result[:errors].count)
     assert_equal('failed', result[:result])
-    assert_equal("Line 1: unknown attribute 'firstname2' for User.", result[:errors][0])
-    assert_equal("Line 2: unknown attribute 'firstname2' for User.", result[:errors][1])
+    assert_equal("Line 1: Unable to create record - unknown attribute 'firstname2' for User.", result[:errors][0])
+    assert_equal("Line 2: Unable to create record - unknown attribute 'firstname2' for User.", result[:errors][1])
 
     assert_nil(User.find_by(login: 'user-invalid-import1'))
     assert_nil(User.find_by(login: 'user-invalid-import2'))