< back

notes

Sidekiq

jid='a5f9c821485ba4c7d2da148f'
Sidekiq::JobSet.find_job(jid)

Sidekiq::Queue.new.delete_by_class(MyWorker)
Sidekiq::RetrySet.new.scan("NoMethodError") { |job| job.delete }
Sidekiq::ScheduledSet.new.scan(jid) { |job| job.delete }
Sidekiq::DeadSet.new.scan("FTPWorker") { |job| ... }

Sidekiq::ScheduledSet.new.find_job(jid)
Sidekiq::RetrySet.new.find_job(jid)
Sidekiq::DeadSet.new.find_job(jid)
Sidekiq::Queue.new(queue_name).find_job(jid)

Sidekiq::Queue.new("default").to_a.map { _1.klass }.tally.sort_by{_2}
Sidekiq::ScheduledSet.new.to_a.map(&:klass).tally.sort_by{_2}
Sidekiq::RetrySet.new.to_a.map(&:klass).tally.sort_by{_2}
Sidekiq::BatchSet.new.to_a

# Reschedule jobs
Sidekiq::ScheduledSet.new.to_a.each { it.reschedule(Time.current) if it.klass == "someclass" }

# List of jobs for batch
Sidekiq::Batch.new('batch_id').jids

* `Sidekiq::Queue.new('default').filter_map { _1.args if _1.klass == 'someclass' }`
Sidekiq::Workers.new.filter_map do |process_id, thread_id, work|
  "#{work.job.jid} #{work.job.klass}: #{work.job.args}" if work.queue == "default"
end
Sidekiq::Workers.new.filter_map do |process_id, thread_id, work|
  "#{work.job.jid} #{work.job.klass}: #{work.job.args}" if work.job.klass == "someclass"
end
Sidekiq::Workers.new.filter_map do |process_id, thread_id, work|
  work.job.klass if process_id.match?("^ampledash-worker-[0-9]")
end.tally
Sidekiq::Workers.new.filter do |process_id, thread_id, work|
  process_id.match?("^ampledash-worker-[0-9]")
end
Sidekiq::Workers.new.each do |process_id, thread_id, work|
  p "still running" if work.job.jid == "9c3713d49241d6f80ba94c42"
  p work if work.job.klass=="someclass"
end; nil

if a queue is overloaded, what is currently running from that queue? also need to check what is currently running on the worker that consumes that queue, since it might be busy with other queues work

Sidekiq::Workers.new.filter { |process_id, thread_id, work| work.queue == "default" }.map { JSON.parse(_3.payload)["class"] }.tally

Ruby/rails

app # idk why we need this in dev, but it needs to load first
Rails.application.routes.url_helpers.dashboard_lead_list('some-id')

(in case you're skeptic of rails routes?)

gem_name = "debug"
`export DEBIAN_FRONTEND="noninteractive"; apt update -y && apt install -y build-essential && gem install #{gem_name}`
gem_folder_path = `gem which #{gem_name}`.strip.toutf8.gsub("#{gem_name}.rb", "")
$: << gem_folder_path
results = Model.insert_all(hashes_or_models, record_timestamps: true, returning: Arel.sql("*"))
models = results.map { |result| Model.instantiate(result) }
j = <<-'JSON'
{"my_json": 1}
JSON
JSON.parse(j)
* `b[reak] <file>:<line>` or `<file> <line>`
  * Set breakpoint on `<file>:<line>`.
* `b[reak] <class>#<name>`
   * Set breakpoint on the method `<class>#<name>`.
* `b[reak] <expr>.<name>`
   * Set breakpoint on the method `<expr>.<name>`.

this is useful to bypass proxy objects and stuff

RSpec

Postgres

SELECT *, (now() - query_start) AS running_time
FROM pg_stat_activity
WHERE (now() - pg_stat_activity.query_start) > interval '1 minutes' AND state='active' ORDER BY running_time DESC;
SELECT DISTINCT ON (url) url, request_duration
FROM logs
ORDER BY url, timestamp DESC
Operation How it's Affected by Dead Tuples (Bloat) Visibility Check Process
Table Scan High Impact. Must read every tuple (live Checks xmin/xmax of every tuple
and dead) from disk and perform a visibility against the transaction snapshot.
check on all of them, discarding the dead ones.
Index Scan Low Impact. The index points directly to the new, Jumps directly to the live tuple's location
live tuple, bypassing all old, dead versions of (ctid) and performs a visibility check only
that row. on that single tuple.
Index-Only Scan Low Impact. Doesn't visit the heap at all if the Checks the Visibility Map first. If the
page is marked "all-visible" in the Visibility Map. page is "all-visible," no heap fetch or
per-tuple check is needed. Otherwise, it
must visit the heap. 
WITH not_visible_pages AS (
    SELECT blkno::bigint
    FROM pg_visibility('orders')
    WHERE NOT all_visible
  ),
  account_rows AS (
    SELECT (ctid::text::point)[0]::bigint AS page
    FROM orders
    WHERE customer_id = '12345'
  )
  SELECT
    page,
    count(*) AS rows_on_page,
    sum(count(*)) OVER () AS total_not_visible,
    (SELECT count(*) FROM account_rows) AS total_rows
  FROM account_rows
  WHERE page IN (SELECT blkno FROM not_visible_pages)
  GROUP BY page
  ORDER BY count(*) DESC;
  Buffers: shared hit=6,286,343 read=76,851
  SubPlan loops: 485,119
SELECT jsonb_build_object(
  '_3', (
    SELECT max(recently_contacted_emails.contacted_date)
    FROM recently_contacted_emails
    WHERE lead_list_entries.email = recently_contacted_emails.email AND ...
  )
)
FROM lead_list_entries
WHERE lead_list_entries.lead_list_id = :lead_list_id

this scales like "for every lead list entry, probe recent contacts", even though most entries won't have one. better shape: start from the sparse side, reduce it, then join it back:

WITH recent_contacts AS MATERIALIZED (
  SELECT recently_contacted_emails.email,
         max(recently_contacted_emails.contacted_date) AS contacted_date
  FROM recently_contacted_emails
  WHERE EXISTS (
      SELECT 1
      FROM lead_list_entries AS lle_exists
      WHERE lle_exists.email = recently_contacted_emails.email AND ...
    ) AND ...
  GROUP BY recently_contacted_emails.email
)
SELECT jsonb_build_object(
  '_3', recent_contacts.contacted_date
)
FROM lead_list_entries AS lle
LEFT JOIN recent_contacts ON recent_contacts.email = lle.email
WHERE lle.lead_list_id = :lead_list_id

rule of thumb: if B is sparse enrichment data for A, precompute relevant B and left join it. don't run a correlated lookup on all rows of A.

Analytics, CDC

what's the best solution to move data from your oltp postgresql into... everywhere else? an olap db, ETL, snowflake?

https://github.com/erezsh/reladiff

Elastic

Fetch a document (get_doc)

def person_doc(id)
  Searchkick.client.get(index: Person.searchkick_index.name, id:)["_source"]
end
doc = person_doc(person_id)
def company_doc(id)
  Searchkick.client.get(index: Company.searchkick_index.name, id:)["_source"]
end
or Person.searchkick_index.retrieve(Person.new(id: person_id))

Update a document directly

Person.search('*', where: { company_domain: "youtube.com"}).pluck(:id).each { |id|
  doc = person_doc(id)
  doc["excluded_domains_accounts"] = []
  Searchkick.client.update(
    index: Person.searchkick_index.name,
    id: person_id,
    body: { doc: doc }
  )
}

By default, simply adding the call 'searchkick' to a model will do an unclever indexing of all fields (but not has_many or belongs_to attributes). In practice, you'll need to customize what gets indexed. This is done by defining a method on your model called search_data

properties in a document (in an index) can have many fields.

kibana dev tools console. explain

profile: include "profile": true in query more results: include "size": 1000 in query

GET _cat/indices?v
PUT /people_staging/_mapping
{
    "properties": {
        "seniorities_classifier": {
          "type": "keyword"
        }
    }
}
Person.search(
  body: {
    size: 1000,
    _source: false,
    query: {
      function_score: {
        query: { match_all: {} },
        random_score: {
          seed: rand(1000000)
        }
      }
    }
  },
  load: false
).response.dig("hits", "hits").map { |hit| hit["_id"] }

React / frontend

CSS

Chrome extension

Honeycomb

k8s

k9s

https://k9scli.io/topics/commands/

Karafka

kafkacat -b 10.74.3.196:9092 -t collected_contacts_reindex -p 6 -o beginning -C -e
kafkacat -b 10.74.3.196:9092 -t collected_contacts_reindex -p 6 -o end -C -e
./kafka-get-offsets.sh  --broker-list 10.74.3.196:9092 --topic collected_contacts_reindex --time -1
./kafka-get-offsets.sh  --broker-list 10.74.3.196:9092 --topic collected_contacts_reindex --time -2
require "avro_turf/messaging"
AVRO = AvroTurf::Messaging.new(
  registry_url: ENV.fetch("KAFKA_SCHEMA_REGISTRY")
)
data = [AVRO.decode(Base64.decode64('string from kafka-ui'))]

Misc

Agents

Claude

StarRocks

pager

pager cut -c 1-1000 SHOW PROCESSLIST\G nopager

profiling

SET enable_query_cache = false; SET query_cache_entry_max_bytes = 0; SET query_cache_entry_max_rows = 0; SET use_page_cache = false; SET skip_page_cache = true; SET skip_local_disk_cache = true; SET enable_profile = true; SELECT get_query_profile(last_query_id());

< back